From 0cff1c88f766cfe553048f5afc27fb411f4eab38 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 15 Aug 2024 12:53:05 +0300 Subject: [PATCH 01/37] Fix UB in blst aggregate verify (#1107) --- crypto/vm/bls.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crypto/vm/bls.cpp b/crypto/vm/bls.cpp index f6ccc275c..ff5179c72 100644 --- a/crypto/vm/bls.cpp +++ b/crypto/vm/bls.cpp @@ -93,12 +93,13 @@ bool aggregate_verify(const std::vector> &pubs_ms return false; } std::unique_ptr pairing = std::make_unique(true, DST); + blst::P2_Affine p2_zero; for (const auto &p : pubs_msgs) { blst::P1_Affine p1(p.first.data(), P1_SIZE); if (!p1.in_group() || p1.is_inf()) { return false; } - pairing->aggregate(&p1, nullptr, (const td::uint8 *)p.second.data(), p.second.size()); + pairing->aggregate(&p1, &p2_zero, (const td::uint8 *)p.second.data(), p.second.size()); } pairing->commit(); blst::P2_Affine p2(sig.data(), P2_SIZE); From 77a816e461f1cb318750ce2ce652e70a5562422a Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 15 Aug 2024 15:25:16 +0300 Subject: [PATCH 02/37] Improve creating channels in adnl (#1108) * Improve creating channels in adnl * Improve handling of cryptographic keys --- adnl/adnl-peer.cpp | 22 +++++++++++++++------ adnl/adnl-peer.hpp | 3 ++- keyring/keyring.cpp | 38 ++++++++++++++++++++---------------- keyring/keyring.hpp | 7 +++---- keys/encryptor.cpp | 22 --------------------- keys/encryptor.h | 22 --------------------- keys/encryptor.hpp | 14 ++++++++++---- keys/keys.cpp | 47 +++++++++++++++++++++++++++++++++++++++++---- keys/keys.hpp | 14 ++++++++++++++ 9 files changed, 109 insertions(+), 80 deletions(-) diff --git a/adnl/adnl-peer.cpp b/adnl/adnl-peer.cpp index febbdac6e..d82486fed 100644 --- a/adnl/adnl-peer.cpp +++ b/adnl/adnl-peer.cpp @@ -269,7 +269,11 @@ void AdnlPeerPairImpl::send_messages_in(std::vector message size_t ptr = 0; bool first = true; do { + respond_with_nop_after_ = td::Timestamp::in(td::Random::fast(1.0, 2.0)); bool try_reinit = try_reinit_at_ && try_reinit_at_.is_in_past(); + if (try_reinit) { + try_reinit_at_ = td::Timestamp::in(td::Random::fast(0.5, 1.5)); + } bool via_channel = channel_ready_ && !try_reinit; size_t s = (via_channel ? channel_packet_header_max_size() : packet_header_max_size()); if (first) { @@ -504,12 +508,6 @@ void AdnlPeerPairImpl::create_channel(pubkeys::Ed25519 pub, td::uint32 date) { void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageCreateChannel &message) { create_channel(message.key(), message.date()); - if (respond_to_channel_create_after_.is_in_past()) { - respond_to_channel_create_after_ = td::Timestamp::in(td::Random::fast(1.0, 2.0)); - std::vector messages; - messages.emplace_back(adnlmessage::AdnlMessageNop{}, 0); - send_messages(std::move(messages)); - } } void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageConfirmChannel &message) { @@ -526,6 +524,7 @@ void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageConfirmChan } void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageCustom &message) { + respond_with_nop(); td::actor::send_closure(local_actor_, &AdnlLocalId::deliver, peer_id_short_, message.data()); } @@ -538,6 +537,7 @@ void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageReinit &mes } void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageQuery &message) { + respond_with_nop(); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), query_id = message.query_id(), flags = static_cast(0)](td::Result R) { if (R.is_error()) { @@ -556,6 +556,7 @@ void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageQuery &mess } void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageAnswer &message) { + respond_with_nop(); auto Q = out_queries_.find(message.query_id()); if (Q == out_queries_.end()) { @@ -573,6 +574,7 @@ void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageAnswer &mes } void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessagePart &message) { + respond_with_nop(); auto size = message.total_size(); if (size > huge_packet_max_size()) { VLOG(ADNL_WARNING) << this << ": dropping too big huge message: size=" << size; @@ -635,6 +637,14 @@ void AdnlPeerPairImpl::delete_query(AdnlQueryId id) { } } +void AdnlPeerPairImpl::respond_with_nop() { + if (respond_with_nop_after_.is_in_past()) { + std::vector messages; + messages.emplace_back(adnlmessage::AdnlMessageNop{}, 0); + send_messages(std::move(messages)); + } +} + void AdnlPeerPairImpl::reinit(td::int32 date) { if (reinit_date_ == 0) { reinit_date_ = date; diff --git a/adnl/adnl-peer.hpp b/adnl/adnl-peer.hpp index e9a5d428e..40c9eb088 100644 --- a/adnl/adnl-peer.hpp +++ b/adnl/adnl-peer.hpp @@ -122,6 +122,7 @@ class AdnlPeerPairImpl : public AdnlPeerPair { } private: + void respond_with_nop(); void reinit(td::int32 date); td::Result, bool>> get_conn(bool direct_only); void create_channel(pubkeys::Ed25519 pub, td::uint32 date); @@ -214,7 +215,7 @@ class AdnlPeerPairImpl : public AdnlPeerPair { pubkeys::Ed25519 channel_pub_; td::int32 channel_pk_date_; td::actor::ActorOwn channel_; - td::Timestamp respond_to_channel_create_after_; + td::Timestamp respond_with_nop_after_; td::uint64 in_seqno_ = 0; td::uint64 out_seqno_ = 0; diff --git a/keyring/keyring.cpp b/keyring/keyring.cpp index 529a6b8b0..0f45879de 100644 --- a/keyring/keyring.cpp +++ b/keyring/keyring.cpp @@ -27,6 +27,16 @@ namespace ton { namespace keyring { +KeyringImpl::PrivateKeyDescr::PrivateKeyDescr(PrivateKey private_key, bool is_temp) + : public_key(private_key.compute_public_key()), is_temp(is_temp) { + auto D = private_key.create_decryptor_async(); + D.ensure(); + decryptor_sign = D.move_as_ok(); + D = private_key.create_decryptor_async(); + D.ensure(); + decryptor_decrypt = D.move_as_ok(); +} + void KeyringImpl::start_up() { if (db_root_.size() > 0) { td::mkdir(db_root_).ensure(); @@ -45,23 +55,19 @@ td::Result KeyringImpl::load_key(PublicKeyHash k auto name = db_root_ + "/" + key_hash.bits256_value().to_hex(); - auto R = td::read_file(td::CSlice{name}); + auto R = td::read_file_secure(td::CSlice{name}); if (R.is_error()) { return R.move_as_error_prefix("key not in db: "); } auto data = R.move_as_ok(); - auto R2 = PrivateKey::import(td::SecureString(data)); + auto R2 = PrivateKey::import(data); R2.ensure(); auto key = R2.move_as_ok(); - auto pub = key.compute_public_key(); - auto short_id = pub.compute_short_id(); + auto desc = std::make_unique(key, false); + auto short_id = desc->public_key.compute_short_id(); CHECK(short_id == key_hash); - - auto D = key.create_decryptor_async(); - D.ensure(); - - return map_.emplace(short_id, std::make_unique(D.move_as_ok(), pub, false)).first->second.get(); + return map_.emplace(short_id, std::move(desc)).first->second.get(); } void KeyringImpl::add_key(PrivateKey key, bool is_temp, td::Promise promise) { @@ -76,10 +82,7 @@ void KeyringImpl::add_key(PrivateKey key, bool is_temp, td::Promise pr if (db_root_.size() == 0) { CHECK(is_temp); } - auto D = key.create_decryptor_async(); - D.ensure(); - - map_.emplace(short_id, std::make_unique(D.move_as_ok(), pub, is_temp)); + map_.emplace(short_id, std::make_unique(key, is_temp)); if (!is_temp && key.exportable()) { auto S = key.export_as_slice(); @@ -139,7 +142,7 @@ void KeyringImpl::sign_message(PublicKeyHash key_hash, td::BufferSlice data, td: if (S.is_error()) { promise.set_error(S.move_as_error()); } else { - td::actor::send_closure(S.move_as_ok()->decryptor, &DecryptorAsync::sign, std::move(data), std::move(promise)); + td::actor::send_closure(S.move_as_ok()->decryptor_sign, &DecryptorAsync::sign, std::move(data), std::move(promise)); } } @@ -161,7 +164,7 @@ void KeyringImpl::sign_add_get_public_key(PublicKeyHash key_hash, td::BufferSlic } promise.set_value(std::pair{R.move_as_ok(), id}); }); - td::actor::send_closure(D->decryptor, &DecryptorAsync::sign, std::move(data), std::move(P)); + td::actor::send_closure(D->decryptor_sign, &DecryptorAsync::sign, std::move(data), std::move(P)); } void KeyringImpl::sign_messages(PublicKeyHash key_hash, std::vector data, @@ -171,7 +174,7 @@ void KeyringImpl::sign_messages(PublicKeyHash key_hash, std::vectordecryptor, &DecryptorAsync::sign_batch, std::move(data), + td::actor::send_closure(S.move_as_ok()->decryptor_sign, &DecryptorAsync::sign_batch, std::move(data), std::move(promise)); } } @@ -182,7 +185,8 @@ void KeyringImpl::decrypt_message(PublicKeyHash key_hash, td::BufferSlice data, if (S.is_error()) { promise.set_error(S.move_as_error()); } else { - td::actor::send_closure(S.move_as_ok()->decryptor, &DecryptorAsync::decrypt, std::move(data), std::move(promise)); + td::actor::send_closure(S.move_as_ok()->decryptor_decrypt, &DecryptorAsync::decrypt, std::move(data), + std::move(promise)); } } diff --git a/keyring/keyring.hpp b/keyring/keyring.hpp index fc67bd0fe..ec658305a 100644 --- a/keyring/keyring.hpp +++ b/keyring/keyring.hpp @@ -30,12 +30,11 @@ namespace keyring { class KeyringImpl : public Keyring { private: struct PrivateKeyDescr { - td::actor::ActorOwn decryptor; + td::actor::ActorOwn decryptor_sign; + td::actor::ActorOwn decryptor_decrypt; PublicKey public_key; bool is_temp; - PrivateKeyDescr(td::actor::ActorOwn decryptor, PublicKey public_key, bool is_temp) - : decryptor(std::move(decryptor)), public_key(public_key), is_temp(is_temp) { - } + PrivateKeyDescr(PrivateKey private_key, bool is_temp); }; public: diff --git a/keys/encryptor.cpp b/keys/encryptor.cpp index 0b93d36fc..8fef9a095 100644 --- a/keys/encryptor.cpp +++ b/keys/encryptor.cpp @@ -29,28 +29,6 @@ namespace ton { -td::Result> Encryptor::create(const ton_api::PublicKey *id) { - td::Result> res; - ton_api::downcast_call( - *const_cast(id), - td::overloaded([&](const ton_api::pub_unenc &obj) { res = std::make_unique(); }, - [&](const ton_api::pub_ed25519 &obj) { res = std::make_unique(obj.key_); }, - [&](const ton_api::pub_overlay &obj) { res = std::make_unique(); }, - [&](const ton_api::pub_aes &obj) { res = std::make_unique(obj.key_); })); - return res; -} - -td::Result> Decryptor::create(const ton_api::PrivateKey *id) { - td::Result> res; - ton_api::downcast_call( - *const_cast(id), - td::overloaded([&](const ton_api::pk_unenc &obj) { res = std::make_unique(); }, - [&](const ton_api::pk_ed25519 &obj) { res = std::make_unique(obj.key_); }, - [&](const ton_api::pk_overlay &obj) { res = std::make_unique(); }, - [&](const ton_api::pk_aes &obj) { res = std::make_unique(obj.key_); })); - return res; -} - td::Result EncryptorEd25519::encrypt(td::Slice data) { TRY_RESULT_PREFIX(pk, td::Ed25519::generate_private_key(), "failed to generate private key: "); TRY_RESULT_PREFIX(pubkey, pk.get_public_key(), "failed to get public key from private: "); diff --git a/keys/encryptor.h b/keys/encryptor.h index 3035a0cec..818c97d63 100644 --- a/keys/encryptor.h +++ b/keys/encryptor.h @@ -31,7 +31,6 @@ class Encryptor { virtual td::Result encrypt(td::Slice data) = 0; virtual td::Status check_signature(td::Slice message, td::Slice signature) = 0; virtual ~Encryptor() = default; - static td::Result> create(const ton_api::PublicKey *id); }; class Decryptor { @@ -40,7 +39,6 @@ class Decryptor { virtual td::Result sign(td::Slice data) = 0; virtual std::vector> sign_batch(std::vector data); virtual ~Decryptor() = default; - static td::Result> create(const ton_api::PrivateKey *id); }; class EncryptorAsync : public td::actor::Actor { @@ -61,16 +59,6 @@ class EncryptorAsync : public td::actor::Actor { void encrypt(td::BufferSlice data, td::Promise promise) { promise.set_result(encryptor_->encrypt(data.as_slice())); } - template - static td::Result> create(T &id) { - TRY_RESULT(d, Encryptor::create(id)); - return td::actor::create_actor("encryptor", std::move(d)); - } - template - static td::Result> create(T *id) { - TRY_RESULT(d, Encryptor::create(id)); - return td::actor::create_actor("encryptor", std::move(d)); - } }; class DecryptorAsync : public td::actor::Actor { @@ -94,16 +82,6 @@ class DecryptorAsync : public td::actor::Actor { } return decryptor_->sign_batch(v); } - template - static td::Result> create(T &id) { - TRY_RESULT(d, Decryptor::create(id)); - return td::actor::create_actor("decryptor", std::move(d)); - } - template - static td::Result> create(T *id) { - TRY_RESULT(d, Decryptor::create(id)); - return td::actor::create_actor("decryptor", std::move(d)); - } }; } // namespace ton diff --git a/keys/encryptor.hpp b/keys/encryptor.hpp index dbe882398..bcc841dc8 100644 --- a/keys/encryptor.hpp +++ b/keys/encryptor.hpp @@ -83,7 +83,7 @@ class EncryptorEd25519 : public Encryptor { td::Result encrypt(td::Slice data) override; td::Status check_signature(td::Slice message, td::Slice signature) override; - EncryptorEd25519(td::Bits256 key) : pub_(td::SecureString(as_slice(key))) { + EncryptorEd25519(const td::Bits256& key) : pub_(td::SecureString(as_slice(key))) { } }; @@ -94,7 +94,7 @@ class DecryptorEd25519 : public Decryptor { public: td::Result decrypt(td::Slice data) override; td::Result sign(td::Slice data) override; - DecryptorEd25519(td::Bits256 key) : pk_(td::SecureString(as_slice(key))) { + DecryptorEd25519(const td::Bits256& key) : pk_(td::SecureString(as_slice(key))) { } }; @@ -129,12 +129,15 @@ class EncryptorAES : public Encryptor { td::Bits256 shared_secret_; public: + ~EncryptorAES() override { + shared_secret_.set_zero_s(); + } td::Result encrypt(td::Slice data) override; td::Status check_signature(td::Slice message, td::Slice signature) override { return td::Status::Error("can no sign channel messages"); } - EncryptorAES(td::Bits256 shared_secret) : shared_secret_(shared_secret) { + EncryptorAES(const td::Bits256& shared_secret) : shared_secret_(shared_secret) { } }; @@ -143,11 +146,14 @@ class DecryptorAES : public Decryptor { td::Bits256 shared_secret_; public: + ~DecryptorAES() override { + shared_secret_.set_zero_s(); + } td::Result decrypt(td::Slice data) override; td::Result sign(td::Slice data) override { return td::Status::Error("can no sign channel messages"); } - DecryptorAES(td::Bits256 shared_secret) : shared_secret_(shared_secret) { + DecryptorAES(const td::Bits256& shared_secret) : shared_secret_(shared_secret) { } }; diff --git a/keys/keys.cpp b/keys/keys.cpp index 7d6c0c2c4..01afb26d2 100644 --- a/keys/keys.cpp +++ b/keys/keys.cpp @@ -21,6 +21,7 @@ #include "td/utils/overloaded.h" #include "tl-utils/tl-utils.hpp" #include "encryptor.h" +#include "encryptor.hpp" #include "crypto/Ed25519.h" namespace ton { @@ -63,12 +64,31 @@ td::Result PublicKey::import(td::Slice s) { return PublicKey{x}; } +td::Result> pubkeys::Ed25519::create_encryptor() const { + return std::make_unique(data_); +} + +td::Result> pubkeys::AES::create_encryptor() const { + return std::make_unique(data_); +} + +td::Result> pubkeys::Unenc::create_encryptor() const { + return std::make_unique(); +} + +td::Result> pubkeys::Overlay::create_encryptor() const { + return std::make_unique(); +} + td::Result> PublicKey::create_encryptor() const { - return Encryptor::create(tl().get()); + td::Result> res; + pub_key_.visit([&](auto &obj) { res = obj.create_encryptor(); }); + return res; } td::Result> PublicKey::create_encryptor_async() const { - return EncryptorAsync::create(tl().get()); + TRY_RESULT(encryptor, create_encryptor()); + return td::actor::create_actor("encryptor", std::move(encryptor)); } bool PublicKey::empty() const { @@ -109,6 +129,22 @@ privkeys::Ed25519::Ed25519(td::Ed25519::PrivateKey key) { data_.as_slice().copy_from(td::Slice(s)); } +td::Result> privkeys::Ed25519::create_decryptor() const { + return std::make_unique(data_); +} + +td::Result> privkeys::AES::create_decryptor() const { + return std::make_unique(data_); +} + +td::Result> privkeys::Unenc::create_decryptor() const { + return std::make_unique(); +} + +td::Result> privkeys::Overlay::create_decryptor() const { + return std::make_unique(); +} + pubkeys::Ed25519::Ed25519(td::Ed25519::PublicKey key) { auto s = key.as_octet_string(); CHECK(s.length() == 32); @@ -188,11 +224,14 @@ tl_object_ptr PrivateKey::tl() const { } td::Result> PrivateKey::create_decryptor() const { - return Decryptor::create(tl().get()); + td::Result> res; + priv_key_.visit([&](auto &obj) { res = obj.create_decryptor(); }); + return res; } td::Result> PrivateKey::create_decryptor_async() const { - return DecryptorAsync::create(tl().get()); + TRY_RESULT(decryptor, create_decryptor()); + return td::actor::create_actor("decryptor", std::move(decryptor)); } } // namespace ton diff --git a/keys/keys.hpp b/keys/keys.hpp index cf883bbe6..72d0845ac 100644 --- a/keys/keys.hpp +++ b/keys/keys.hpp @@ -110,6 +110,7 @@ class Ed25519 { tl_object_ptr tl() const { return create_tl_object(data_); } + td::Result> create_encryptor() const; bool operator==(const Ed25519 &with) const { return data_ == with.data_; } @@ -141,6 +142,7 @@ class AES { tl_object_ptr tl() const { return create_tl_object(data_); } + td::Result> create_encryptor() const; bool operator==(const AES &with) const { return data_ == with.data_; } @@ -172,6 +174,7 @@ class Unenc { tl_object_ptr tl() const { return create_tl_object(data_.clone_as_buffer_slice()); } + td::Result> create_encryptor() const; bool operator==(const Unenc &with) const { return data_.as_slice() == with.data_.as_slice(); } @@ -203,6 +206,7 @@ class Overlay { tl_object_ptr tl() const { return create_tl_object(data_.clone_as_buffer_slice()); } + td::Result> create_encryptor() const; bool operator==(const Overlay &with) const { return data_.as_slice() == with.data_.as_slice(); } @@ -223,6 +227,9 @@ class PublicKey { td::uint32 serialized_size() const { UNREACHABLE(); } + td::Result> create_encryptor() const { + UNREACHABLE(); + } bool operator==(const Empty &with) const { return false; } @@ -316,6 +323,7 @@ class Ed25519 { } tl_object_ptr pub_tl() const; pubkeys::Ed25519 pub() const; + td::Result> create_decryptor() const; static Ed25519 random(); }; @@ -359,6 +367,7 @@ class AES { pubkeys::AES pub() const { return pubkeys::AES{data_}; } + td::Result> create_decryptor() const; }; class Unenc { @@ -393,6 +402,7 @@ class Unenc { pubkeys::Unenc pub() const { return pubkeys::Unenc{data_.clone()}; } + td::Result> create_decryptor() const; }; class Overlay { @@ -427,6 +437,7 @@ class Overlay { pubkeys::Overlay pub() const { return pubkeys::Overlay{data_.clone()}; } + td::Result> create_decryptor() const; }; } // namespace privkeys @@ -450,6 +461,9 @@ class PrivateKey { PublicKey pub() const { UNREACHABLE(); } + td::Result> create_decryptor() const { + UNREACHABLE(); + } }; td::Variant priv_key_{Empty{}}; From 9661676646f059e1103d2b3cab4bd33712de53ea Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 15 Aug 2024 15:26:35 +0300 Subject: [PATCH 03/37] Improve dht lookup in overlays (#1104) Continue dht lookup even if value was found --- dht/dht-in.hpp | 1 + dht/dht-query.cpp | 34 +++++++++++++++++++++++--- dht/dht-query.hpp | 51 +++++++++++++++++++++++++++++++++------ dht/dht.cpp | 17 ++++++++++++- dht/dht.h | 1 + overlay/overlay.cpp | 49 +++++++++++++++++++------------------ overlay/overlay.hpp | 3 ++- tdutils/td/utils/Status.h | 6 +++++ 8 files changed, 127 insertions(+), 35 deletions(-) diff --git a/dht/dht-in.hpp b/dht/dht-in.hpp index c2d20455f..0d668d438 100644 --- a/dht/dht-in.hpp +++ b/dht/dht-in.hpp @@ -179,6 +179,7 @@ class DhtMemberImpl : public DhtMember { void get_value(DhtKey key, td::Promise result) override { get_value_in(key.compute_key_id(), std::move(result)); } + void get_value_many(DhtKey key, std::function callback, td::Promise promise) override; void alarm() override { alarm_timestamp() = td::Timestamp::in(1.0); diff --git a/dht/dht-query.cpp b/dht/dht-query.cpp index b84ef8c37..3d43b1069 100644 --- a/dht/dht-query.cpp +++ b/dht/dht-query.cpp @@ -210,8 +210,11 @@ void DhtQueryFindValue::on_result(td::Result R, adnl::AdnlNodeI send_get_nodes = true; return; } - promise_.set_value(std::move(value)); - need_stop = true; + if (on_value_found(std::move(value))) { + send_get_nodes = true; + } else { + need_stop = true; + } }, [&](ton_api::dht_valueNotFound &v) { add_nodes(DhtNodesList{std::move(v.nodes_), our_network_id()}); @@ -244,7 +247,32 @@ void DhtQueryFindValue::on_result_nodes(td::Result R, adnl::Adn } void DhtQueryFindValue::finish(DhtNodesList list) { - promise_.set_error(td::Status::Error(ErrorCode::notready, "dht key not found")); +} + +bool DhtQueryFindValueSingle::on_value_found(DhtValue value) { + promise_.set_value(std::move(value)); + found_ = true; + return false; +} + +void DhtQueryFindValueSingle::tear_down() { + if (!found_) { + promise_.set_error(td::Status::Error(ErrorCode::notready, "dht key not found")); + } +} + +bool DhtQueryFindValueMany::on_value_found(DhtValue value) { + callback_(std::move(value)); + found_ = true; + return true; +} + +void DhtQueryFindValueMany::tear_down() { + if (found_) { + promise_.set_value(td::Unit()); + } else { + promise_.set_error(td::Status::Error(ErrorCode::notready, "dht key not found")); + } } DhtQueryStore::DhtQueryStore(DhtValue key_value, DhtMember::PrintId print_id, adnl::AdnlNodeIdShort src, diff --git a/dht/dht-query.hpp b/dht/dht-query.hpp index e47403618..e305f107a 100644 --- a/dht/dht-query.hpp +++ b/dht/dht-query.hpp @@ -126,16 +126,11 @@ class DhtQueryFindNodes : public DhtQuery { }; class DhtQueryFindValue : public DhtQuery { - private: - td::Promise promise_; - public: DhtQueryFindValue(DhtKeyId key, DhtMember::PrintId print_id, adnl::AdnlNodeIdShort src, DhtNodesList list, td::uint32 k, td::uint32 a, td::int32 our_network_id, DhtNode self, bool client_only, - td::actor::ActorId node, td::actor::ActorId adnl, - td::Promise promise) - : DhtQuery(key, print_id, src, k, a, our_network_id, std::move(self), client_only, node, adnl) - , promise_(std::move(promise)) { + td::actor::ActorId node, td::actor::ActorId adnl) + : DhtQuery(key, print_id, src, k, a, our_network_id, std::move(self), client_only, node, adnl) { add_nodes(std::move(list)); } void send_one_query(adnl::AdnlNodeIdShort id) override; @@ -146,6 +141,48 @@ class DhtQueryFindValue : public DhtQuery { std::string get_name() const override { return "find value"; } + + virtual bool on_value_found(DhtValue value) = 0; +}; + +class DhtQueryFindValueSingle : public DhtQueryFindValue { + public: + DhtQueryFindValueSingle(DhtKeyId key, DhtMember::PrintId print_id, adnl::AdnlNodeIdShort src, DhtNodesList list, + td::uint32 k, td::uint32 a, td::int32 our_network_id, DhtNode self, bool client_only, + td::actor::ActorId node, td::actor::ActorId adnl, + td::Promise promise) + : DhtQueryFindValue(key, print_id, src, std::move(list), k, a, our_network_id, std::move(self), client_only, node, + adnl) + , promise_(std::move(promise)) { + add_nodes(std::move(list)); + } + bool on_value_found(DhtValue value) override; + void tear_down() override; + + private: + td::Promise promise_; + bool found_ = false; +}; + +class DhtQueryFindValueMany : public DhtQueryFindValue { + public: + DhtQueryFindValueMany(DhtKeyId key, DhtMember::PrintId print_id, adnl::AdnlNodeIdShort src, DhtNodesList list, + td::uint32 k, td::uint32 a, td::int32 our_network_id, DhtNode self, bool client_only, + td::actor::ActorId node, td::actor::ActorId adnl, + std::function callback, td::Promise promise) + : DhtQueryFindValue(key, print_id, src, std::move(list), k, a, our_network_id, std::move(self), client_only, node, + adnl) + , callback_(std::move(callback)) + , promise_(std::move(promise)) { + add_nodes(std::move(list)); + } + bool on_value_found(DhtValue value) override; + void tear_down() override; + + private: + std::function callback_; + td::Promise promise_; + bool found_ = false; }; class DhtQueryStore : public td::actor::Actor { diff --git a/dht/dht.cpp b/dht/dht.cpp index 8d7b02b7d..b774b5f91 100644 --- a/dht/dht.cpp +++ b/dht/dht.cpp @@ -470,7 +470,7 @@ void DhtMemberImpl::get_value_in(DhtKeyId key, td::Promise result) { network_id = network_id_, id = id_, client_only = client_only_](td::Result R) mutable { R.ensure(); - td::actor::create_actor("FindValueQuery", key, print_id, id, std::move(list), k, a, network_id, + td::actor::create_actor("FindValueQuery", key, print_id, id, std::move(list), k, a, network_id, R.move_as_ok(), client_only, SelfId, adnl, std::move(promise)) .release(); }); @@ -478,6 +478,21 @@ void DhtMemberImpl::get_value_in(DhtKeyId key, td::Promise result) { get_self_node(std::move(P)); } +void DhtMemberImpl::get_value_many(DhtKey key, std::function callback, td::Promise promise) { + DhtKeyId key_id = key.compute_key_id(); + auto P = td::PromiseCreator::lambda( + [key = key_id, callback = std::move(callback), promise = std::move(promise), SelfId = actor_id(this), + print_id = print_id(), adnl = adnl_, list = get_nearest_nodes(key_id, k_ * 2), k = k_, a = a_, + network_id = network_id_, id = id_, client_only = client_only_](td::Result R) mutable { + R.ensure(); + td::actor::create_actor("FindValueManyQuery", key, print_id, id, std::move(list), k, a, + network_id, R.move_as_ok(), client_only, SelfId, adnl, + std::move(callback), std::move(promise)) + .release(); + }); + get_self_node(std::move(P)); +} + void DhtMemberImpl::register_reverse_connection(adnl::AdnlNodeIdFull client, td::Promise promise) { auto client_short = client.compute_short_id(); td::uint32 ttl = (td::uint32)td::Clocks::system() + 300; diff --git a/dht/dht.h b/dht/dht.h index b9c65c8a7..5abff94a1 100644 --- a/dht/dht.h +++ b/dht/dht.h @@ -53,6 +53,7 @@ class Dht : public td::actor::Actor { virtual void set_value(DhtValue key_value, td::Promise result) = 0; virtual void get_value(DhtKey key, td::Promise result) = 0; + virtual void get_value_many(DhtKey key, std::function callback, td::Promise promise) = 0; virtual void register_reverse_connection(adnl::AdnlNodeIdFull client, td::Promise promise) = 0; virtual void request_reverse_ping(adnl::AdnlNode target, adnl::AdnlNodeIdShort client, diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index f964ccc5c..43b3b7e5a 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -283,11 +283,14 @@ void OverlayImpl::alarm() { } if (next_dht_query_ && next_dht_query_.is_in_past()) { next_dht_query_ = td::Timestamp::never(); - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result res) { - td::actor::send_closure(SelfId, &OverlayImpl::receive_dht_nodes, std::move(res), true); - }); - td::actor::send_closure(dht_node_, &dht::Dht::get_value, dht::DhtKey{overlay_id_.pubkey_hash(), "nodes", 0}, - std::move(P)); + std::function callback = [SelfId = actor_id(this)](dht::DhtValue value) { + td::actor::send_closure(SelfId, &OverlayImpl::receive_dht_nodes, std::move(value)); + }; + td::Promise on_finish = [SelfId = actor_id(this)](td::Result R) { + td::actor::send_closure(SelfId, &OverlayImpl::dht_lookup_finished, R.move_as_status()); + }; + td::actor::send_closure(dht_node_, &dht::Dht::get_value_many, dht::DhtKey{overlay_id_.pubkey_hash(), "nodes", 0}, + std::move(callback), std::move(on_finish)); } if (update_db_at_.is_in_past()) { if (peers_.size() > 0) { @@ -311,30 +314,30 @@ void OverlayImpl::alarm() { } } -void OverlayImpl::receive_dht_nodes(td::Result res, bool dummy) { +void OverlayImpl::receive_dht_nodes(dht::DhtValue v) { CHECK(public_); - if (res.is_ok()) { - auto v = res.move_as_ok(); - auto R = fetch_tl_object(v.value().clone(), true); - if (R.is_ok()) { - auto r = R.move_as_ok(); - VLOG(OVERLAY_INFO) << this << ": received " << r->nodes_.size() << " nodes from overlay"; - VLOG(OVERLAY_EXTRA_DEBUG) << this << ": nodes: " << ton_api::to_string(r); - std::vector nodes; - for (auto &n : r->nodes_) { - auto N = OverlayNode::create(n); - if (N.is_ok()) { - nodes.emplace_back(N.move_as_ok()); - } + auto R = fetch_tl_object(v.value().clone(), true); + if (R.is_ok()) { + auto r = R.move_as_ok(); + VLOG(OVERLAY_INFO) << this << ": received " << r->nodes_.size() << " nodes from overlay"; + VLOG(OVERLAY_EXTRA_DEBUG) << this << ": nodes: " << ton_api::to_string(r); + std::vector nodes; + for (auto &n : r->nodes_) { + auto N = OverlayNode::create(n); + if (N.is_ok()) { + nodes.emplace_back(N.move_as_ok()); } - add_peers(std::move(nodes)); - } else { - VLOG(OVERLAY_WARNING) << this << ": incorrect value in DHT for overlay nodes: " << R.move_as_error(); } + add_peers(std::move(nodes)); } else { - VLOG(OVERLAY_NOTICE) << this << ": can not get value from DHT: " << res.move_as_error(); + VLOG(OVERLAY_WARNING) << this << ": incorrect value in DHT for overlay nodes: " << R.move_as_error(); } +} +void OverlayImpl::dht_lookup_finished(td::Status S) { + if (S.is_error()) { + VLOG(OVERLAY_NOTICE) << this << ": can not get value from DHT: " << S; + } if (!(next_dht_store_query_ && next_dht_store_query_.is_in_past())) { finish_dht_query(); return; diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index 90fcc43d7..8fb3d91d7 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -166,7 +166,8 @@ class OverlayImpl : public Overlay { certs_[key] = std::move(cert); } - void receive_dht_nodes(td::Result res, bool dummy); + void receive_dht_nodes(dht::DhtValue v); + void dht_lookup_finished(td::Status S); void update_dht_nodes(OverlayNode node); void update_neighbours(td::uint32 nodes_to_change); diff --git a/tdutils/td/utils/Status.h b/tdutils/td/utils/Status.h index 9c4f1b7df..8bc210dba 100644 --- a/tdutils/td/utils/Status.h +++ b/tdutils/td/utils/Status.h @@ -555,6 +555,12 @@ class Result { }; return status_.move_as_error_suffix(suffix); } + Status move_as_status() TD_WARN_UNUSED_RESULT { + if (status_.is_error()) { + return move_as_error(); + } + return Status::OK(); + } const T &ok() const { LOG_CHECK(status_.is_ok()) << status_; return value_; From 06515c3735ddb2b551845c624cb9f01379403213 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 16 Aug 2024 10:23:41 +0300 Subject: [PATCH 04/37] Limit the number of threads to 127 (#1111) --- dht-server/dht-server.cpp | 12 ++++++++---- validator-engine/validator-engine.cpp | 12 ++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/dht-server/dht-server.cpp b/dht-server/dht-server.cpp index 025cf7d51..eb183cad6 100644 --- a/dht-server/dht-server.cpp +++ b/dht-server/dht-server.cpp @@ -1231,15 +1231,19 @@ int main(int argc, char *argv[]) { }); td::uint32 threads = 7; p.add_checked_option( - 't', "threads", PSTRING() << "number of threads (default=" << threads << ")", [&](td::Slice fname) { + 't', "threads", PSTRING() << "number of threads (default=" << threads << ")", [&](td::Slice arg) { td::int32 v; try { - v = std::stoi(fname.str()); + v = std::stoi(arg.str()); } catch (...) { return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: not a number"); } - if (v < 1 || v > 256) { - return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: should be in range [1..256]"); + if (v <= 0) { + return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: should be > 0"); + } + if (v > 127) { + LOG(WARNING) << "`--threads " << v << "` is too big, effective value will be 127"; + v = 127; } threads = v; return td::Status::OK(); diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 88cef8d49..a9a3b21d4 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -4157,15 +4157,19 @@ int main(int argc, char *argv[]) { }); td::uint32 threads = 7; p.add_checked_option( - 't', "threads", PSTRING() << "number of threads (default=" << threads << ")", [&](td::Slice fname) { + 't', "threads", PSTRING() << "number of threads (default=" << threads << ")", [&](td::Slice arg) { td::int32 v; try { - v = std::stoi(fname.str()); + v = std::stoi(arg.str()); } catch (...) { return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: not a number"); } - if (v < 1 || v > 256) { - return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: should be in range [1..256]"); + if (v <= 0) { + return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: should be > 0"); + } + if (v > 127) { + LOG(WARNING) << "`--threads " << v << "` is too big, effective value will be 127"; + v = 127; } threads = v; return td::Status::OK(); From 5bdcb5e2ce987dc68c9aefd8fba89d7fc3b051ca Mon Sep 17 00:00:00 2001 From: krigga Date: Tue, 20 Aug 2024 19:50:59 +0300 Subject: [PATCH 05/37] Disable testing and fuzzing for openssl when building WASM packages (#1116) Co-authored-by: EmelyanenkoK --- assembly/wasm/fift-func-wasm-build-ubuntu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assembly/wasm/fift-func-wasm-build-ubuntu.sh b/assembly/wasm/fift-func-wasm-build-ubuntu.sh index e7a54d16f..2d3507b23 100644 --- a/assembly/wasm/fift-func-wasm-build-ubuntu.sh +++ b/assembly/wasm/fift-func-wasm-build-ubuntu.sh @@ -85,7 +85,7 @@ cd .. if [ ! -f "openssl/openssl_em" ]; then cd openssl make clean - emconfigure ./Configure linux-generic32 no-shared no-dso no-engine no-unit-test + emconfigure ./Configure linux-generic32 no-shared no-dso no-engine no-unit-test no-tests no-fuzz-afl no-fuzz-libfuzzer sed -i 's/CROSS_COMPILE=.*/CROSS_COMPILE=/g' Makefile sed -i 's/-ldl//g' Makefile sed -i 's/-O3/-Os/g' Makefile From 9c3dc22b781b666640032ec8aa79c61331931726 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 20 Aug 2024 19:54:16 +0300 Subject: [PATCH 06/37] Improve validator session stats (#1117) * Improve validator session stats * Collator stats: block limits, number of processed external messages * Collator and validator work time * Last key block seqno * Approvers and signers * End validator session stats --- catchain/catchain.h | 1 + catchain/catchain.hpp | 9 ++ tdutils/td/utils/Timer.cpp | 43 +++++++++ tdutils/td/utils/Timer.h | 18 ++++ tl/generate/scheme/ton_api.tl | 25 +++-- tl/generate/scheme/ton_api.tlo | Bin 91004 -> 92472 bytes validator-session/validator-session-types.h | 14 +++ validator-session/validator-session.cpp | 29 ++++++ validator-session/validator-session.h | 1 + validator-session/validator-session.hpp | 1 + validator/impl/collator-impl.h | 5 + validator/impl/collator.cpp | 63 ++++++++++++ validator/impl/validate-query.cpp | 21 ++++ validator/impl/validate-query.hpp | 4 + validator/interfaces/validator-manager.h | 17 ++++ validator/manager-disk.hpp | 3 + validator/manager-hardfork.hpp | 3 + validator/manager.cpp | 102 +++++++++++++++++--- validator/manager.hpp | 19 +++- validator/validator-group.cpp | 11 +++ validator/validator-group.hpp | 11 ++- 21 files changed, 373 insertions(+), 27 deletions(-) diff --git a/catchain/catchain.h b/catchain/catchain.h index 912957e56..c5c8af28d 100644 --- a/catchain/catchain.h +++ b/catchain/catchain.h @@ -96,6 +96,7 @@ class CatChain : public td::actor::Actor { virtual void send_query_via(const PublicKeyHash &dst, std::string name, td::Promise promise, td::Timestamp timeout, td::BufferSlice query, td::uint64 max_answer_size, td::actor::ActorId via) = 0; + virtual void get_source_heights(td::Promise> promise) = 0; virtual void destroy() = 0; static td::actor::ActorOwn create(std::unique_ptr callback, const CatChainOptions &opts, diff --git a/catchain/catchain.hpp b/catchain/catchain.hpp index 8c8bb99ae..586cf4744 100644 --- a/catchain/catchain.hpp +++ b/catchain/catchain.hpp @@ -115,6 +115,15 @@ class CatChainImpl : public CatChain { td::actor::send_closure(receiver_, &CatChainReceiverInterface::send_custom_query_data_via, dst, name, std::move(promise), timeout, std::move(query), max_answer_size, via); } + void get_source_heights(td::Promise> promise) override { + std::vector heights(top_source_blocks_.size(), 0); + for (size_t i = 0; i < top_source_blocks_.size(); ++i) { + if (top_source_blocks_[i]) { + heights[i] = top_source_blocks_[i]->height(); + } + } + promise.set_result(std::move(heights)); + } void destroy() override; CatChainImpl(std::unique_ptr callback, const CatChainOptions &opts, td::actor::ActorId keyring, td::actor::ActorId adnl, diff --git a/tdutils/td/utils/Timer.cpp b/tdutils/td/utils/Timer.cpp index 1f72fba96..24de099aa 100644 --- a/tdutils/td/utils/Timer.cpp +++ b/tdutils/td/utils/Timer.cpp @@ -91,4 +91,47 @@ double PerfWarningTimer::elapsed() const { return Time::now() - start_at_; } +static double thread_cpu_clock() { +#if defined(CLOCK_THREAD_CPUTIME_ID) + timespec ts; + int result = clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts); + CHECK(result == 0); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1e-9; +#else + return 0.0; // TODO: MacOS and Windows support (currently cpu timer is used only in validators) +#endif +} + +ThreadCpuTimer::ThreadCpuTimer(bool is_paused) : is_paused_(is_paused) { + if (is_paused_) { + start_time_ = 0; + } else { + start_time_ = thread_cpu_clock(); + } +} + +void ThreadCpuTimer::pause() { + if (is_paused_) { + return; + } + elapsed_ += thread_cpu_clock() - start_time_; + is_paused_ = true; +} + +void ThreadCpuTimer::resume() { + if (!is_paused_) { + return; + } + start_time_ = thread_cpu_clock(); + is_paused_ = false; +} + +double ThreadCpuTimer::elapsed() const { + double res = elapsed_; + if (!is_paused_) { + res += thread_cpu_clock() - start_time_; + } + return res; +} + } // namespace td diff --git a/tdutils/td/utils/Timer.h b/tdutils/td/utils/Timer.h index 3e0cafbf5..a27cac8a7 100644 --- a/tdutils/td/utils/Timer.h +++ b/tdutils/td/utils/Timer.h @@ -62,4 +62,22 @@ class PerfWarningTimer { std::function callback_; }; +class ThreadCpuTimer { + public: + ThreadCpuTimer() : ThreadCpuTimer(false) { + } + explicit ThreadCpuTimer(bool is_paused); + ThreadCpuTimer(const ThreadCpuTimer &other) = default; + ThreadCpuTimer &operator=(const ThreadCpuTimer &other) = default; + + double elapsed() const; + void pause(); + void resume(); + + private: + double elapsed_{0}; + double start_time_; + bool is_paused_{false}; +}; + } // namespace td diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index b33ca5425..bf919b0fd 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -388,8 +388,8 @@ tonNode.newShardBlock block:tonNode.blockIdExt cc_seqno:int data:bytes = tonNode tonNode.blockBroadcastCompressed.data signatures:(vector tonNode.blockSignature) proof_data:bytes = tonNode.blockBroadcaseCompressed.Data; -tonNode.blockBroadcast id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int - signatures:(vector tonNode.blockSignature) +tonNode.blockBroadcast id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int + signatures:(vector tonNode.blockSignature) proof:bytes data:bytes = tonNode.Broadcast; tonNode.blockBroadcastCompressed id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int flags:# compressed:bytes = tonNode.Broadcast; @@ -769,13 +769,19 @@ http.server.config dhs:(vector http.server.dnsEntry) local_hosts:(vector http.se ---types--- -validatorSession.statsProducer id:int256 candidate_id:int256 block_status:int comment:string - block_timestamp:double is_accepted:Bool is_ours:Bool got_submit_at:double +validatorSession.collationStats bytes:int gas:int lt_delta:int cat_bytes:int cat_gas:int cat_lt_delta:int + limits_log:string ext_msgs_total:int ext_msgs_filtered:int ext_msgs_accepted:int ext_msgs_rejected:int = validadorSession.CollationStats; + +validatorSession.statsProducer id:int256 candidate_id:int256 block_status:int root_hash:int256 file_hash:int256 + comment:string block_timestamp:double is_accepted:Bool is_ours:Bool got_submit_at:double collation_time:double collated_at:double collation_cached:Bool + collation_work_time:double collation_cpu_work_time:double + collation_stats:validatorSession.collationStats validation_time:double validated_at:double validation_cached:Bool + validation_work_time:double validation_cpu_work_time:double gen_utime:double - approved_weight:long approved_33pct_at:double approved_66pct_at:double - signed_weight:long signed_33pct_at:double signed_66pct_at:double + approved_weight:long approved_33pct_at:double approved_66pct_at:double approvers:string + signed_weight:long signed_33pct_at:double signed_66pct_at:double signers:string serialize_time:double deserialize_time:double serialized_size:int = validatorSession.StatsProducer; validatorSession.statsRound timestamp:double producers:(vector validatorSession.statsProducer) = validatorSession.StatsRound; @@ -786,9 +792,14 @@ validatorSession.stats success:Bool id:tonNode.blockIdExt timestamp:double self: first_round:int rounds:(vector validatorSession.statsRound) = validatorSession.Stats; validatorSession.newValidatorGroupStats.node id:int256 weight:long = validatorSession.newValidatorGroupStats.Node; -validatorSession.newValidatorGroupStats session_id:int256 workchain:int shard:long cc_seqno:int timestamp:double +validatorSession.newValidatorGroupStats session_id:int256 workchain:int shard:long cc_seqno:int + last_key_block_seqno:int timestamp:double self_idx:int nodes:(vector validatorSession.newValidatorGroupStats.node) = validatorSession.NewValidatorGroupStats; +validatorSession.endValidatorGroupStats.node id:int256 catchain_blocks:int = validatorSession.endValidatorGroupStats.Node; +validatorSession.endValidatorGroupStats session_id:int256 timestamp:double + nodes:(vector validatorSession.endValidatorGroupStats.node) = validatorSession.EndValidatorGroupStats; + ---functions--- ---types--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index da1aa331d2048803f582b99ede705270f28a22ea..337dd071e3f746cd822236e2dcb63c194b379de4 100644 GIT binary patch delta 841 zcmex!jCIE)R^CUm^{p77fNLY~d3}|4min3UWr;bNDTyihMZu}X#hLkedd~SdIf*4e zR&YsTN%6!D5}WVn?_iXi_R1(;9iq4dq1ZJqW%@;aM&Zdn3?(K{F|=S=WW;%J@&&^c zES-~cgeKP*6-c&d`z+GI(3lF;xKM&oWU{2O{Ny)AKNx#AKQ*>L#R_)Q^ouNv!ivbw z014kbCgj5n^nmoU`4q$f(n!LI$;qh&B^bg*sadJX5MlAj2^JhIAonkp_$P=X7(t-{3Pu)C zC{I^#VpN!JAjBvz{e~c;%H$QkDiWYTWQ0auku#ePhi%(Iuv#8|#x<;+`kvg1oKb5kb=%#)B1M@S?Wl%h(?@*^aRL1_o%UXVjUo@boC zRg|$^R0Nt3fXNF`3spvW|5Mycsh;*50=|J@O1?0~Q`Bp4O&r$K2JQ0PuR5U3%*4@@b^8Ht&B@ku%P T$=St|57^3pWtZ1TFnR+3<5Dl9 delta 209 zcmdmSiS^GhR^CUm^{p77fPEwHdHu~220Iuhix^q3>_0DYW^#bh3KmawcJ;}Bj0z_2 zF#f^VwRxk7{i&^27{3TipCZJ#fEA=neezyobxsiL<}o23_UQ*j8MP-@6uC@)Bg&{U zT}Onm0c_g#4I+#mm{>vP$xT-jWfYsfTbz-T6T}DUlbAd)NO}4PamIk{9TJQ>jMFWo p7;C0WNHZ>({y>_sVtR!P&>WEB>3?JxJ0PqMSw@BJE^>@tya3`PN1gxx diff --git a/validator-session/validator-session-types.h b/validator-session/validator-session-types.h index e13c36d24..78a9b2460 100644 --- a/validator-session/validator-session-types.h +++ b/validator-session/validator-session-types.h @@ -77,6 +77,8 @@ struct ValidatorSessionStats { ValidatorSessionCandidateId candidate_id = ValidatorSessionCandidateId::zero(); int block_status = status_none; double block_timestamp = -1.0; + td::Bits256 root_hash = td::Bits256::zero(); + td::Bits256 file_hash = td::Bits256::zero(); std::string comment; bool is_accepted = false; @@ -159,11 +161,23 @@ struct NewValidatorGroupStats { ValidatorSessionId session_id = ValidatorSessionId::zero(); ShardIdFull shard{masterchainId}; CatchainSeqno cc_seqno = 0; + BlockSeqno last_key_block_seqno = 0; double timestamp = -1.0; td::uint32 self_idx = 0; std::vector nodes; }; +struct EndValidatorGroupStats { + struct Node { + PublicKeyHash id = PublicKeyHash::zero(); + td::uint32 catchain_blocks = 0; + }; + + ValidatorSessionId session_id = ValidatorSessionId::zero(); + double timestamp = -1.0; + std::vector nodes; +}; + } // namespace validatorsession } // namespace ton diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 46dd44403..be5443785 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -270,6 +270,8 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice } stat->deserialize_time = deserialize_time; stat->serialized_size = data.size(); + stat->root_hash = candidate->root_hash_; + stat->file_hash = file_hash; } if ((td::int32)block_round < (td::int32)cur_round_ - MAX_PAST_ROUND_BLOCK || @@ -468,6 +470,8 @@ void ValidatorSessionImpl::generated_block(td::uint32 round, ValidatorSessionCan stat->collated_at = td::Clocks::system(); stat->block_timestamp = td::Clocks::system(); stat->collation_cached = collation_cached; + stat->root_hash = root_hash; + stat->file_hash = file_hash; } if (round != cur_round_) { return; @@ -602,6 +606,8 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { if (stat->block_timestamp <= 0.0) { stat->block_timestamp = td::Clocks::system(); } + stat->root_hash = B->root_hash_; + stat->file_hash = td::sha256_bits256(B->data_); } auto P = td::PromiseCreator::lambda([round = cur_round_, hash = block_id, root_hash = block->get_root_hash(), @@ -997,6 +1003,29 @@ void ValidatorSessionImpl::get_current_stats(td::Promise promise.set_result(cur_stats_); } +void ValidatorSessionImpl::get_end_stats(td::Promise promise) { + if (!started_) { + promise.set_error(td::Status::Error(ErrorCode::notready, "not started")); + return; + } + EndValidatorGroupStats stats; + stats.session_id = unique_hash_; + stats.timestamp = td::Clocks::system(); + stats.nodes.resize(description().get_total_nodes()); + for (size_t i = 0; i < stats.nodes.size(); ++i) { + stats.nodes[i].id = description().get_source_id(i); + } + td::actor::send_closure(catchain_, &catchain::CatChain::get_source_heights, + [promise = std::move(promise), + stats = std::move(stats)](td::Result> R) mutable { + TRY_RESULT_PROMISE(promise, heights, std::move(R)); + for (size_t i = 0; i < std::min(heights.size(), stats.nodes.size()); ++i) { + stats.nodes[i].catchain_blocks = heights[i]; + } + promise.set_result(std::move(stats)); + }); +} + void ValidatorSessionImpl::get_validator_group_info_for_litequery( td::uint32 cur_round, td::Promise>> promise) { diff --git a/validator-session/validator-session.h b/validator-session/validator-session.h index 0870f6718..2e1ed9b13 100644 --- a/validator-session/validator-session.h +++ b/validator-session/validator-session.h @@ -105,6 +105,7 @@ class ValidatorSession : public td::actor::Actor { virtual void start() = 0; virtual void destroy() = 0; virtual void get_current_stats(td::Promise promise) = 0; + virtual void get_end_stats(td::Promise promise) = 0; virtual void get_validator_group_info_for_litequery( td::uint32 cur_round, td::Promise>> promise) = 0; diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp index 580582824..2ee4885b9 100644 --- a/validator-session/validator-session.hpp +++ b/validator-session/validator-session.hpp @@ -187,6 +187,7 @@ class ValidatorSessionImpl : public ValidatorSession { void start() override; void destroy() override; void get_current_stats(td::Promise promise) override; + void get_end_stats(td::Promise promise) override; void get_validator_group_info_for_litequery( td::uint32 cur_round, td::Promise>> promise) override; diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 913a0ed87..b8d9e56d3 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -356,6 +356,11 @@ class Collator final : public td::actor::Actor { public: static td::uint32 get_skip_externals_queue_size(); + + private: + td::Timer work_timer_{true}; + td::ThreadCpuTimer cpu_work_timer_{true}; + CollationStats stats_; }; } // namespace validator diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index c6dd7caf2..f465c0f55 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -1772,6 +1772,12 @@ bool Collator::register_shard_block_creators(std::vector creator_li * @returns True if collation is successful, false otherwise. */ bool Collator::try_collate() { + work_timer_.resume(); + cpu_work_timer_.resume(); + SCOPE_EXIT { + work_timer_.pause(); + cpu_work_timer_.pause(); + }; if (!preinit_complete) { LOG(WARNING) << "running do_preinit()"; if (!do_preinit()) { @@ -3481,6 +3487,29 @@ bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalT return true; } +/** + * Creates a string that explains which limit is exceeded. Used for collator stats. + * + * @param block_limit_status Status of block limits. + * @param cls Which limit class is exceeded. + * + * @returns String for collator stats. + */ +static std::string block_full_comment(const block::BlockLimitStatus& block_limit_status, unsigned cls) { + auto bytes = block_limit_status.estimate_block_size(); + if (!block_limit_status.limits.bytes.fits(cls, bytes)) { + return PSTRING() << "block_full bytes " << bytes; + } + if (!block_limit_status.limits.gas.fits(cls, block_limit_status.gas_used)) { + return PSTRING() << "block_full gas " << block_limit_status.gas_used; + } + auto lt_delta = block_limit_status.cur_lt - block_limit_status.limits.start_lt; + if (!block_limit_status.limits.lt_delta.fits(cls, lt_delta)) { + return PSTRING() << "block_full lt_delta " << lt_delta; + } + return ""; +} + /** * Processes inbound internal messages from message queues of the neighbors. * Messages are processed until the normal limit is reached, soft timeout is reached or there are no more messages. @@ -3495,11 +3524,14 @@ bool Collator::process_inbound_internal_messages() { block_full_ = !block_limit_status_->fits(block::ParamLimits::cl_normal); if (block_full_) { LOG(INFO) << "BLOCK FULL, stop processing inbound internal messages"; + stats_.limits_log += PSTRING() << "INBOUND_INT_MESSAGES: " + << block_full_comment(*block_limit_status_, block::ParamLimits::cl_normal) << "\n"; break; } if (soft_timeout_.is_in_past(td::Timestamp::now())) { block_full_ = true; LOG(WARNING) << "soft timeout reached, stop processing inbound internal messages"; + stats_.limits_log += PSTRING() << "INBOUND_INT_MESSAGES: timeout\n"; break; } auto kv = nb_out_msgs_->extract_cur(); @@ -3547,15 +3579,23 @@ bool Collator::process_inbound_external_messages() { } if (full) { LOG(INFO) << "BLOCK FULL, stop processing external messages"; + stats_.limits_log += PSTRING() << "INBOUND_EXT_MESSAGES: " + << block_full_comment(*block_limit_status_, block::ParamLimits::cl_soft) << "\n"; break; } if (medium_timeout_.is_in_past(td::Timestamp::now())) { LOG(WARNING) << "medium timeout reached, stop processing inbound external messages"; + stats_.limits_log += PSTRING() << "INBOUND_EXT_MESSAGES: timeout\n"; break; } auto ext_msg = ext_msg_struct.cell; ton::Bits256 hash{ext_msg->get_hash().bits()}; int r = process_external_message(std::move(ext_msg)); + if (r > 0) { + ++stats_.ext_msgs_accepted; + } else { + ++stats_.ext_msgs_rejected; + } if (r < 0) { bad_ext_msgs_.emplace_back(ext_msg_struct.hash); return false; @@ -3661,11 +3701,15 @@ bool Collator::process_dispatch_queue() { block_full_ = !block_limit_status_->fits(block::ParamLimits::cl_normal); if (block_full_) { LOG(INFO) << "BLOCK FULL, stop processing dispatch queue"; + stats_.limits_log += PSTRING() << "DISPATCH_QUEUE_STAGE_" << iter << ": " + << block_full_comment(*block_limit_status_, block::ParamLimits::cl_normal) + << "\n"; return true; } if (soft_timeout_.is_in_past(td::Timestamp::now())) { block_full_ = true; LOG(WARNING) << "soft timeout reached, stop processing dispatch queue"; + stats_.limits_log += PSTRING() << "DISPATCH_QUEUE_STAGE_" << iter << ": timeout\n"; return true; } StdSmcAddress src_addr; @@ -3715,6 +3759,7 @@ bool Collator::process_dispatch_queue() { ++total_count; if (total_count >= max_total_count[iter]) { dispatch_queue_total_limit_reached_ = true; + stats_.limits_log += PSTRING() << "DISPATCH_QUEUE_STAGE_" << iter << ": total limit reached\n"; break; } } @@ -4064,6 +4109,8 @@ bool Collator::process_new_messages(bool enqueue_only) { if ((block_full_ || have_unprocessed_account_dispatch_queue_) && !enqueue_only) { LOG(INFO) << "BLOCK FULL, enqueue all remaining new messages"; enqueue_only = true; + stats_.limits_log += PSTRING() << "NEW_MESSAGES: " + << block_full_comment(*block_limit_status_, block::ParamLimits::cl_normal) << "\n"; } LOG(DEBUG) << "have message with lt=" << msg.lt; int res = process_one_new_message(std::move(msg), enqueue_only); @@ -4072,6 +4119,8 @@ bool Collator::process_new_messages(bool enqueue_only) { } else if (res == 3) { LOG(INFO) << "All remaining new messages must be enqueued (BLOCK FULL)"; enqueue_only = true; + stats_.limits_log += PSTRING() << "NEW_MESSAGES: " + << block_full_comment(*block_limit_status_, block::ParamLimits::cl_normal) << "\n"; } } return true; @@ -5435,6 +5484,18 @@ bool Collator::create_block_candidate() { td::actor::send_closure_later(manager, &ValidatorManager::complete_external_messages, std::move(delay_ext_msgs_), std::move(bad_ext_msgs_)); } + + double work_time = work_timer_.elapsed(); + double cpu_work_time = cpu_work_timer_.elapsed(); + LOG(WARNING) << "Collate query work time = " << work_time << "s, cpu time = " << cpu_work_time << "s"; + stats_.bytes = block_limit_status_->estimate_block_size(); + stats_.gas = block_limit_status_->gas_used; + stats_.lt_delta = block_limit_status_->cur_lt - block_limit_status_->limits.start_lt; + stats_.cat_bytes = block_limit_status_->limits.classify_size(stats_.bytes); + stats_.cat_gas = block_limit_status_->limits.classify_gas(stats_.gas); + stats_.cat_lt_delta = block_limit_status_->limits.classify_lt(block_limit_status_->cur_lt); + td::actor::send_closure(manager, &ValidatorManager::record_collate_query_stats, block_candidate->id, work_time, + cpu_work_time, std::move(stats_)); return true; } @@ -5539,6 +5600,7 @@ void Collator::after_get_external_messages(td::Result ext_msg_cell = ext_msg->root_cell(); @@ -5550,6 +5612,7 @@ void Collator::after_get_external_messages(td::Resulthash()); } } diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 8c39a1ab4..003b7f9f7 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -118,6 +118,7 @@ bool ValidateQuery::reject_query(std::string error, td::BufferSlice reason) { error = error_ctx() + error; LOG(ERROR) << "REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error; if (main_promise) { + record_stats(); errorlog::ErrorLog::log(PSTRING() << "REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error << ": data=" << block_candidate.id.file_hash.to_hex() << " collated_data=" << block_candidate.collated_file_hash.to_hex()); @@ -155,6 +156,7 @@ bool ValidateQuery::soft_reject_query(std::string error, td::BufferSlice reason) error = error_ctx() + error; LOG(ERROR) << "SOFT REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error; if (main_promise) { + record_stats(); errorlog::ErrorLog::log(PSTRING() << "SOFT REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error << ": data=" << block_candidate.id.file_hash.to_hex() << " collated_data=" << block_candidate.collated_file_hash.to_hex()); @@ -177,6 +179,7 @@ bool ValidateQuery::fatal_error(td::Status error) { error.ensure_error(); LOG(ERROR) << "aborting validation of block candidate for " << shard_.to_str() << " : " << error.to_string(); if (main_promise) { + record_stats(); auto c = error.code(); if (c <= -667 && c >= -670) { errorlog::ErrorLog::log(PSTRING() << "FATAL ERROR: aborting validation of block candidate for " << shard_.to_str() @@ -234,6 +237,7 @@ bool ValidateQuery::fatal_error(std::string err_msg, int err_code) { */ void ValidateQuery::finish_query() { if (main_promise) { + record_stats(); LOG(WARNING) << "validate query done"; main_promise.set_result(now_); } @@ -6764,6 +6768,12 @@ bool ValidateQuery::try_validate() { if (pending) { return true; } + work_timer_.resume(); + cpu_work_timer_.resume(); + SCOPE_EXIT { + work_timer_.pause(); + cpu_work_timer_.pause(); + }; try { if (!stage_) { LOG(WARNING) << "try_validate stage 0"; @@ -6903,6 +6913,17 @@ void ValidateQuery::written_candidate() { finish_query(); } +/** + * Sends validation work time to manager. + */ +void ValidateQuery::record_stats() { + double work_time = work_timer_.elapsed(); + double cpu_work_time = cpu_work_timer_.elapsed(); + LOG(WARNING) << "Validate query work time = " << work_time << "s, cpu time = " << cpu_work_time << "s"; + td::actor::send_closure(manager, &ValidatorManager::record_validate_query_stats, block_candidate.id, work_time, + cpu_work_time); +} + } // namespace validator } // namespace ton diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 824afb49d..104950938 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -398,6 +398,10 @@ class ValidateQuery : public td::actor::Actor { } return true; } + + td::Timer work_timer_{true}; + td::ThreadCpuTimer cpu_work_timer_{true}; + void record_stats(); }; } // namespace validator diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 0e9fab73b..b6016bc2b 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -52,6 +52,16 @@ struct AsyncSerializerState { UnixTime last_written_block_ts; }; +struct CollationStats { + td::uint32 bytes, gas, lt_delta; + int cat_bytes, cat_gas, cat_lt_delta; + std::string limits_log; + td::uint32 ext_msgs_total = 0; + td::uint32 ext_msgs_filtered = 0; + td::uint32 ext_msgs_accepted = 0; + td::uint32 ext_msgs_rejected = 0; +}; + using ValidateCandidateResult = td::Variant; class ValidatorManager : public ValidatorManagerInterface { @@ -173,6 +183,7 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) = 0; virtual void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) = 0; + virtual void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) = 0; virtual void get_block_handle_for_litequery(BlockIdExt block_id, td::Promise promise) = 0; virtual void get_block_data_for_litequery(BlockIdExt block_id, td::Promise> promise) = 0; @@ -192,6 +203,12 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void add_lite_query_stats(int lite_query_id) { } + virtual void record_collate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, + CollationStats stats) { + } + virtual void record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) { + } + static bool is_persistent_state(UnixTime ts, UnixTime prev_ts) { return ts / (1 << 17) != prev_ts / (1 << 17); } diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index a77be2725..3a77f2301 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -388,6 +388,9 @@ class ValidatorManagerImpl : public ValidatorManager { void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override { UNREACHABLE(); } + void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) override { + UNREACHABLE(); + } void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { queue_size_counter_ = diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index e7175b77b..cf4d3799f 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -450,6 +450,9 @@ class ValidatorManagerImpl : public ValidatorManager { void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override { UNREACHABLE(); } + void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) override { + UNREACHABLE(); + } void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { queue_size_counter_ = diff --git a/validator/manager.cpp b/validator/manager.cpp index eb082d91e..fa592a788 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -44,6 +44,7 @@ #include "td/utils/JsonBuilder.h" #include "common/delay.h" +#include "td/utils/filesystem.h" #include "validator/stats-merger.h" @@ -2044,7 +2045,7 @@ void ValidatorManagerImpl::update_shards() { } new_validator_groups_.emplace(val_group_id, std::move(it2->second)); } else { - auto G = create_validator_group(val_group_id, shard, val_set, opts, started_); + auto G = create_validator_group(val_group_id, shard, val_set, key_seqno, opts, started_); if (!G.empty()) { td::actor::send_closure(G, &ValidatorGroup::start, prev, last_masterchain_block_id_, last_masterchain_state_->get_unix_time()); @@ -2100,7 +2101,7 @@ void ValidatorManagerImpl::update_shards() { } new_validator_groups_.emplace(val_group_id, std::move(it2->second)); } else { - auto G = create_validator_group(val_group_id, shard, val_set, opts, started_); + auto G = create_validator_group(val_group_id, shard, val_set, key_seqno, opts, started_); if (!G.empty()) { td::actor::send_closure(G, &ValidatorGroup::start, prev, last_masterchain_block_id_, last_masterchain_state_->get_unix_time()); @@ -2127,7 +2128,7 @@ void ValidatorManagerImpl::update_shards() { } else { new_next_validator_groups_.emplace( val_group_id, - ValidatorGroupEntry{create_validator_group(val_group_id, shard, val_set, opts, started_), shard}); + ValidatorGroupEntry{create_validator_group(val_group_id, shard, val_set, key_seqno, opts, started_), shard}); } } } @@ -2230,7 +2231,7 @@ ValidatorSessionId ValidatorManagerImpl::get_validator_set_id(ShardIdFull shard, } td::actor::ActorOwn ValidatorManagerImpl::create_validator_group( - ValidatorSessionId session_id, ShardIdFull shard, td::Ref validator_set, + ValidatorSessionId session_id, ShardIdFull shard, td::Ref validator_set, BlockSeqno key_seqno, validatorsession::ValidatorSessionOptions opts, bool init_session) { if (check_gc_list_.count(session_id) == 1) { return td::actor::ActorOwn{}; @@ -2241,8 +2242,8 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group auto validator_id = get_validator(shard, validator_set); CHECK(!validator_id.is_zero()); auto G = td::actor::create_actor( - "validatorgroup", shard, validator_id, session_id, validator_set, opts, keyring_, adnl_, rldp_, overlays_, - db_root_, actor_id(this), init_session, + "validatorgroup", shard, validator_id, session_id, validator_set, key_seqno, opts, keyring_, adnl_, rldp_, + overlays_, db_root_, actor_id(this), init_session, opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), opts_); return G; } @@ -2831,13 +2832,35 @@ void ValidatorManagerImpl::log_validator_session_stats(BlockIdExt block_id, for (const auto &round : stats.rounds) { std::vector> producers; for (const auto &producer : round.producers) { + BlockIdExt cur_block_id{block_id.id, producer.root_hash, producer.file_hash}; + auto it = recorded_block_stats_.find(cur_block_id); + tl_object_ptr collation_stats; + if (it != recorded_block_stats_.end() && it->second.collator_stats_) { + auto &stats = it->second.collator_stats_.value(); + collation_stats = create_tl_object( + stats.bytes, stats.gas, stats.lt_delta, stats.cat_bytes, stats.cat_gas, stats.cat_lt_delta, + stats.limits_log, stats.ext_msgs_total, stats.ext_msgs_filtered, stats.ext_msgs_accepted, + stats.ext_msgs_rejected); + } + std::string approvers, signers; + for (bool x : producer.approvers) { + approvers += (x ? '1' : '0'); + } + for (bool x : producer.signers) { + signers += (x ? '1' : '0'); + } producers.push_back(create_tl_object( - producer.id.bits256_value(), producer.candidate_id, producer.block_status, producer.comment, - producer.block_timestamp, producer.is_accepted, producer.is_ours, producer.got_submit_at, - producer.collation_time, producer.collated_at, producer.collation_cached, producer.validation_time, - producer.validated_at, producer.validation_cached, producer.gen_utime, producer.approved_weight, - producer.approved_33pct_at, producer.approved_66pct_at, producer.signed_weight, producer.signed_33pct_at, - producer.signed_66pct_at, producer.serialize_time, producer.deserialize_time, producer.serialized_size)); + producer.id.bits256_value(), producer.candidate_id, producer.block_status, producer.root_hash, + producer.file_hash, producer.comment, producer.block_timestamp, producer.is_accepted, producer.is_ours, + producer.got_submit_at, producer.collation_time, producer.collated_at, producer.collation_cached, + it == recorded_block_stats_.end() ? -1.0 : it->second.collator_work_time_, + it == recorded_block_stats_.end() ? -1.0 : it->second.collator_cpu_work_time_, std::move(collation_stats), + producer.validation_time, producer.validated_at, producer.validation_cached, + it == recorded_block_stats_.end() ? -1.0 : it->second.validator_work_time_, + it == recorded_block_stats_.end() ? -1.0 : it->second.validator_cpu_work_time_, producer.gen_utime, + producer.approved_weight, producer.approved_33pct_at, producer.approved_66pct_at, std::move(approvers), + producer.signed_weight, producer.signed_33pct_at, producer.signed_66pct_at, std::move(signers), + producer.serialize_time, producer.deserialize_time, producer.serialized_size)); } rounds.push_back(create_tl_object(round.timestamp, std::move(producers))); } @@ -2869,8 +2892,8 @@ void ValidatorManagerImpl::log_new_validator_group_stats(validatorsession::NewVa create_tl_object(node.id.bits256_value(), node.weight)); } auto obj = create_tl_object( - stats.session_id, stats.shard.workchain, stats.shard.shard, stats.cc_seqno, stats.timestamp, stats.self_idx, - std::move(nodes)); + stats.session_id, stats.shard.workchain, stats.shard.shard, stats.cc_seqno, stats.last_key_block_seqno, + stats.timestamp, stats.self_idx, std::move(nodes)); auto s = td::json_encode(td::ToJson(*obj.get()), false); s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); @@ -2879,7 +2902,31 @@ void ValidatorManagerImpl::log_new_validator_group_stats(validatorsession::NewVa file << s << "\n"; file.close(); - LOG(INFO) << "Writing new validator group stats for " << stats.shard.to_str(); + LOG(INFO) << "Writing new validator group stats for " << stats.session_id << " shard=" << stats.shard.to_str() + << " cc_seqno=" << stats.cc_seqno; +} + +void ValidatorManagerImpl::log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) { + std::string fname = opts_->get_session_logs_file(); + if (fname.empty()) { + return; + } + std::vector> nodes; + for (const auto &node : stats.nodes) { + nodes.push_back(create_tl_object(node.id.bits256_value(), + node.catchain_blocks)); + } + auto obj = create_tl_object(stats.session_id, stats.timestamp, + std::move(nodes)); + auto s = td::json_encode(td::ToJson(*obj.get()), false); + s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); + + std::ofstream file; + file.open(fname, std::ios_base::app); + file << s << "\n"; + file.close(); + + LOG(INFO) << "Writing end validator group stats for " << stats.session_id; } void ValidatorManagerImpl::get_block_handle_for_litequery(BlockIdExt block_id, td::Promise promise) { @@ -3165,6 +3212,31 @@ td::actor::ActorOwn ValidatorManagerFactory::create( rldp, overlays); } +void ValidatorManagerImpl::record_collate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, + CollationStats stats) { + auto &record = new_block_stats_record(block_id); + record.collator_work_time_ = work_time; + record.collator_cpu_work_time_ = cpu_work_time; + record.collator_stats_ = std::move(stats); +} + +void ValidatorManagerImpl::record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) { + auto &record = new_block_stats_record(block_id); + record.validator_work_time_ = work_time; + record.validator_cpu_work_time_ = cpu_work_time; +} + +ValidatorManagerImpl::RecordedBlockStats &ValidatorManagerImpl::new_block_stats_record(BlockIdExt block_id) { + if (!recorded_block_stats_.count(block_id)) { + recorded_block_stats_lru_.push(block_id); + if (recorded_block_stats_lru_.size() > 4096) { + recorded_block_stats_.erase(recorded_block_stats_lru_.front()); + recorded_block_stats_lru_.pop(); + } + } + return recorded_block_stats_[block_id]; +} + size_t ValidatorManagerImpl::CheckedExtMsgCounter::get_msg_count(WorkchainId wc, StdSmcAddress addr) { before_query(); auto it1 = counter_cur_.find({wc, addr}); diff --git a/validator/manager.hpp b/validator/manager.hpp index 12354c634..99aa4e0e1 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -38,6 +38,7 @@ #include #include #include +#include namespace ton { @@ -261,7 +262,7 @@ class ValidatorManagerImpl : public ValidatorManager { BlockSeqno last_key_block_seqno, const validatorsession::ValidatorSessionOptions &opts); td::actor::ActorOwn create_validator_group(ValidatorSessionId session_id, ShardIdFull shard, - td::Ref validator_set, + td::Ref validator_set, BlockSeqno key_seqno, validatorsession::ValidatorSessionOptions opts, bool create_catchain); struct ValidatorGroupEntry { @@ -589,6 +590,7 @@ class ValidatorManagerImpl : public ValidatorManager { void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override; void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override; + void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) override; void update_options(td::Ref opts) override; @@ -708,6 +710,21 @@ class ValidatorManagerImpl : public ValidatorManager { td::uint32 ls_stats_check_ext_messages_{0}; td::actor::ActorOwn candidates_buffer_; + + struct RecordedBlockStats { + double collator_work_time_ = -1.0; + double collator_cpu_work_time_ = -1.0; + td::optional collator_stats_; + double validator_work_time_ = -1.0; + double validator_cpu_work_time_ = -1.0; + }; + std::map recorded_block_stats_; + std::queue recorded_block_stats_lru_; + + void record_collate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, + CollationStats stats) override; + void record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) override; + RecordedBlockStats &new_block_stats_record(BlockIdExt block_id); }; } // namespace validator diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index fc3ebe541..4b61c07cd 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -386,6 +386,7 @@ void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterch stats.session_id = session_id_; stats.shard = shard_; stats.cc_seqno = validator_set_->get_catchain_seqno(); + stats.last_key_block_seqno = last_key_block_seqno_; stats.timestamp = td::Clocks::system(); td::uint32 idx = 0; for (const auto& node : validator_set_->export_vector()) { @@ -417,6 +418,16 @@ void ValidatorGroup::destroy() { td::actor::send_closure(manager, &ValidatorManager::log_validator_session_stats, block_id, std::move(stats)); }); + td::actor::send_closure(session_, &validatorsession::ValidatorSession::get_end_stats, + [manager = manager_](td::Result R) { + if (R.is_error()) { + LOG(DEBUG) << "Failed to get validator session end stats: " << R.move_as_error(); + return; + } + auto stats = R.move_as_ok(); + td::actor::send_closure(manager, &ValidatorManager::log_end_validator_group_stats, + std::move(stats)); + }); auto ses = session_.release(); delay_action([ses]() mutable { td::actor::send_closure(ses, &validatorsession::ValidatorSession::destroy); }, td::Timestamp::in(10.0)); diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index 3499da9d7..936d2fdc7 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -69,15 +69,17 @@ class ValidatorGroup : public td::actor::Actor { } ValidatorGroup(ShardIdFull shard, PublicKeyHash local_id, ValidatorSessionId session_id, - td::Ref validator_set, validatorsession::ValidatorSessionOptions config, - td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId overlays, - std::string db_root, td::actor::ActorId validator_manager, bool create_session, + td::Ref validator_set, BlockSeqno last_key_block_seqno, + validatorsession::ValidatorSessionOptions config, td::actor::ActorId keyring, + td::actor::ActorId adnl, td::actor::ActorId rldp, + td::actor::ActorId overlays, std::string db_root, + td::actor::ActorId validator_manager, bool create_session, bool allow_unsafe_self_blocks_resync, td::Ref opts) : shard_(shard) , local_id_(std::move(local_id)) , session_id_(session_id) , validator_set_(std::move(validator_set)) + , last_key_block_seqno_(last_key_block_seqno) , config_(std::move(config)) , keyring_(keyring) , adnl_(adnl) @@ -115,6 +117,7 @@ class ValidatorGroup : public td::actor::Actor { UnixTime min_ts_; td::Ref validator_set_; + BlockSeqno last_key_block_seqno_; validatorsession::ValidatorSessionOptions config_; td::actor::ActorId keyring_; From 9a10f79fbacdcd6a51e193f5c19f082cccca4884 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 20 Aug 2024 19:55:01 +0300 Subject: [PATCH 07/37] Fix getting creator stats in lite-client (#1115) --- lite-client/lite-client.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index 020aca705..02a5fab67 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -3429,9 +3429,7 @@ void TestNode::got_creator_stats(ton::BlockIdExt req_blkid, ton::BlockIdExt blki promise.set_error(td::Status::Error(PSLICE() << "invalid CreatorStats record with key " << key.to_hex())); return; } - if (mc_cnt.modified_since(min_utime) || shard_cnt.modified_since(min_utime)) { - func(key, mc_cnt, shard_cnt); - } + func(key, mc_cnt, shard_cnt); allow_eq = false; } if (complete) { From 908415d00b706cfab913ad727147d6dd5926c88a Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 23 Aug 2024 11:46:40 +0300 Subject: [PATCH 08/37] Accelerator, part 1 (#1119) This commit contains some parts of https://github.com/ton-blockchain/ton/tree/accelerator This is auxiliary code that mostly does not change node behavior. 1) Semiprivate overlays and other improvements in overlays code 2) Rename actual_min_split -> monitor_min_split, fix building shard overlays 3) Loading block candidates by block id from DB, fix accept_block after validator restart 4) Cells: ProofStorageStat and changes in CellUsageTree 5) Remove some unused code, other minor changes --- CMakeLists.txt | 2 + adnl/adnl-query.cpp | 2 +- crypto/block/block.tlb | 8 +- crypto/block/mc-config.cpp | 2 +- crypto/block/mc-config.h | 2 +- crypto/vm/boc.cpp | 31 + crypto/vm/boc.h | 12 + crypto/vm/cells/CellUsageTree.cpp | 12 +- crypto/vm/cells/CellUsageTree.h | 13 +- crypto/vm/cells/MerkleProof.h | 4 + crypto/vm/cells/UsageCell.h | 2 +- keys/keys.hpp | 4 + overlay/overlay-broadcast.cpp | 5 +- overlay/overlay-fec-broadcast.cpp | 2 +- overlay/overlay-id.hpp | 100 ++- overlay/overlay-manager.cpp | 299 +++++++-- overlay/overlay-manager.h | 27 +- overlay/overlay-peers.cpp | 596 +++++++++++++++--- overlay/overlay.cpp | 341 ++++++---- overlay/overlay.h | 41 +- overlay/overlay.hpp | 243 ++++--- overlay/overlays.h | 139 +++- test/test-overlay.cpp | 450 +++++++++++++ test/test-validator-session-state.cpp | 4 + tl-utils/common-utils.hpp | 33 + tl/generate/scheme/ton_api.tl | 18 +- tl/generate/scheme/ton_api.tlo | Bin 92472 -> 94788 bytes ton/ton-tl.hpp | 11 +- ton/ton-types.h | 2 + validator-session/candidate-serializer.cpp | 57 +- validator-session/candidate-serializer.h | 7 +- .../validator-session-description.cpp | 5 + .../validator-session-description.h | 1 + .../validator-session-description.hpp | 1 + validator-session/validator-session.cpp | 1 + validator/db/fileref.cpp | 25 + validator/db/fileref.hpp | 37 +- validator/db/rootdb.cpp | 37 +- validator/db/rootdb.hpp | 1 + validator/db/statedb.hpp | 3 - validator/fabric.h | 8 +- validator/impl/CMakeLists.txt | 1 - validator/impl/accept-block.cpp | 29 + validator/impl/accept-block.hpp | 2 + validator/impl/collate-query-impl.h | 63 -- validator/impl/collator-impl.h | 8 +- validator/impl/collator.cpp | 10 +- validator/impl/collator.h | 19 - validator/impl/fabric.cpp | 21 +- validator/impl/shard.cpp | 13 +- validator/impl/shard.hpp | 3 +- validator/impl/validate-query.cpp | 22 +- validator/impl/validate-query.hpp | 6 +- validator/interfaces/db.h | 1 + validator/interfaces/shard.h | 3 +- validator/manager-disk.cpp | 9 +- validator/manager-disk.hpp | 1 + validator/manager-hardfork.cpp | 5 + validator/manager-hardfork.hpp | 1 + validator/manager.cpp | 22 +- validator/manager.hpp | 1 + validator/net/download-block-new.cpp | 4 +- validator/shard-client.cpp | 2 +- validator/validator-group.cpp | 13 +- validator/validator-group.hpp | 5 +- validator/validator.h | 1 + 66 files changed, 2218 insertions(+), 635 deletions(-) create mode 100644 test/test-overlay.cpp delete mode 100644 validator/impl/collate-query-impl.h diff --git a/CMakeLists.txt b/CMakeLists.txt index b92ff6f1b..70cb8b8d8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -529,6 +529,8 @@ target_link_libraries(test-rldp2 adnl adnltest dht rldp2 tl_api) add_executable(test-validator-session-state test/test-validator-session-state.cpp) target_link_libraries(test-validator-session-state adnl dht rldp validatorsession tl_api) +add_executable(test-overlay test/test-overlay.cpp) +target_link_libraries(test-overlay overlay tdutils tdactor adnl adnltest tl_api dht ) add_executable(test-catchain test/test-catchain.cpp) target_link_libraries(test-catchain overlay tdutils tdactor adnl adnltest rldp tl_api dht catchain ) diff --git a/adnl/adnl-query.cpp b/adnl/adnl-query.cpp index e098c1344..89dbbfc49 100644 --- a/adnl/adnl-query.cpp +++ b/adnl/adnl-query.cpp @@ -25,7 +25,7 @@ namespace ton { namespace adnl { void AdnlQuery::alarm() { - set_error(td::Status::Error(ErrorCode::timeout, "adnl query timeout")); + set_error(td::Status::Error(ErrorCode::timeout, PSTRING() << "timeout for adnl query " << name_)); } void AdnlQuery::result(td::BufferSlice data) { promise_.set_value(std::move(data)); diff --git a/crypto/block/block.tlb b/crypto/block/block.tlb index a3684f563..6f9754267 100644 --- a/crypto/block/block.tlb +++ b/crypto/block/block.tlb @@ -666,15 +666,15 @@ wc_split_merge_timings#0 //workchain#a5 enabled_since:uint32 min_split:(## 8) max_split:(## 8) // { min_split <= max_split } { max_split <= 60 } -workchain#a6 enabled_since:uint32 actual_min_split:(## 8) - min_split:(## 8) max_split:(## 8) { actual_min_split <= min_split } +workchain#a6 enabled_since:uint32 monitor_min_split:(## 8) + min_split:(## 8) max_split:(## 8) { monitor_min_split <= min_split } basic:(## 1) active:Bool accept_msgs:Bool flags:(## 13) { flags = 0 } zerostate_root_hash:bits256 zerostate_file_hash:bits256 version:uint32 format:(WorkchainFormat basic) = WorkchainDescr; -workchain_v2#a7 enabled_since:uint32 actual_min_split:(## 8) - min_split:(## 8) max_split:(## 8) { actual_min_split <= min_split } +workchain_v2#a7 enabled_since:uint32 monitor_min_split:(## 8) + min_split:(## 8) max_split:(## 8) { monitor_min_split <= min_split } basic:(## 1) active:Bool accept_msgs:Bool flags:(## 13) { flags = 0 } zerostate_root_hash:bits256 zerostate_file_hash:bits256 version:uint32 format:(WorkchainFormat basic) diff --git a/crypto/block/mc-config.cpp b/crypto/block/mc-config.cpp index 6da0f034d..d682271fb 100644 --- a/crypto/block/mc-config.cpp +++ b/crypto/block/mc-config.cpp @@ -2075,7 +2075,7 @@ bool WorkchainInfo::unpack(ton::WorkchainId wc, vm::CellSlice& cs) { } auto unpack_v1 = [this](auto& info) { enabled_since = info.enabled_since; - actual_min_split = info.actual_min_split; + monitor_min_split = info.monitor_min_split; min_split = info.min_split; max_split = info.max_split; basic = info.basic; diff --git a/crypto/block/mc-config.h b/crypto/block/mc-config.h index 25d94ec7e..94ab291bf 100644 --- a/crypto/block/mc-config.h +++ b/crypto/block/mc-config.h @@ -414,7 +414,7 @@ struct CatchainValidatorsConfig { struct WorkchainInfo : public td::CntObject { ton::WorkchainId workchain{ton::workchainInvalid}; ton::UnixTime enabled_since; - td::uint32 actual_min_split; + td::uint32 monitor_min_split; td::uint32 min_split, max_split; bool basic; bool active; diff --git a/crypto/vm/boc.cpp b/crypto/vm/boc.cpp index bd334cbfc..3e15f62b6 100644 --- a/crypto/vm/boc.cpp +++ b/crypto/vm/boc.cpp @@ -1214,4 +1214,35 @@ bool VmStorageStat::add_storage(const CellSlice& cs) { return true; } +static td::uint64 estimate_prunned_size() { + return 41; +} + +static td::uint64 estimate_serialized_size(const Ref& cell) { + return cell->get_serialized_size() + cell->size_refs() * 3 + 3; +} + +void ProofStorageStat::add_cell(const Ref& cell) { + auto& status = cells_[cell->get_hash()]; + if (status == c_loaded) { + return; + } + if (status == c_prunned) { + proof_size_ -= estimate_prunned_size(); + } + status = c_loaded; + proof_size_ += estimate_serialized_size(cell); + for (unsigned i = 0; i < cell->size_refs(); ++i) { + auto& child_status = cells_[cell->get_ref(i)->get_hash()]; + if (child_status == c_none) { + child_status = c_prunned; + proof_size_ += estimate_prunned_size(); + } + } +} + +td::uint64 ProofStorageStat::estimate_proof_size() const { + return proof_size_; +} + } // namespace vm diff --git a/crypto/vm/boc.h b/crypto/vm/boc.h index a5f87774d..bdd0a06f1 100644 --- a/crypto/vm/boc.h +++ b/crypto/vm/boc.h @@ -163,6 +163,18 @@ struct VmStorageStat { } }; +class ProofStorageStat { + public: + void add_cell(const Ref& cell); + td::uint64 estimate_proof_size() const; + private: + enum CellStatus { + c_none = 0, c_prunned = 1, c_loaded = 2 + }; + std::map cells_; + td::uint64 proof_size_ = 0; +}; + struct CellSerializationInfo { bool special; Cell::LevelMask level_mask; diff --git a/crypto/vm/cells/CellUsageTree.cpp b/crypto/vm/cells/CellUsageTree.cpp index 3f43ec6bd..410b3fcd6 100644 --- a/crypto/vm/cells/CellUsageTree.cpp +++ b/crypto/vm/cells/CellUsageTree.cpp @@ -22,12 +22,12 @@ namespace vm { // // CellUsageTree::NodePtr // -bool CellUsageTree::NodePtr::on_load() const { +bool CellUsageTree::NodePtr::on_load(const td::Ref& cell) const { auto tree = tree_weak_.lock(); if (!tree) { return false; } - tree->on_load(node_id_); + tree->on_load(node_id_, cell); return true; } @@ -111,8 +111,14 @@ void CellUsageTree::set_use_mark_for_is_loaded(bool use_mark) { use_mark_ = use_mark; } -void CellUsageTree::on_load(NodeId node_id) { +void CellUsageTree::on_load(NodeId node_id, const td::Ref& cell) { + if (nodes_[node_id].is_loaded) { + return; + } nodes_[node_id].is_loaded = true; + if (cell_load_callback_) { + cell_load_callback_(cell); + } } CellUsageTree::NodeId CellUsageTree::create_child(NodeId node_id, unsigned ref_id) { diff --git a/crypto/vm/cells/CellUsageTree.h b/crypto/vm/cells/CellUsageTree.h index 150dd2bda..af0f21f53 100644 --- a/crypto/vm/cells/CellUsageTree.h +++ b/crypto/vm/cells/CellUsageTree.h @@ -22,8 +22,12 @@ #include "td/utils/int_types.h" #include "td/utils/logging.h" +#include namespace vm { + +class DataCell; + class CellUsageTree : public std::enable_shared_from_this { public: using NodeId = td::uint32; @@ -38,7 +42,7 @@ class CellUsageTree : public std::enable_shared_from_this { return node_id_ == 0 || tree_weak_.expired(); } - bool on_load() const; + bool on_load(const td::Ref& cell) const; NodePtr create_child(unsigned ref_id) const; bool mark_path(CellUsageTree* master_tree) const; bool is_from_tree(const CellUsageTree* master_tree) const; @@ -59,6 +63,10 @@ class CellUsageTree : public std::enable_shared_from_this { void set_use_mark_for_is_loaded(bool use_mark = true); NodeId create_child(NodeId node_id, unsigned ref_id); + void set_cell_load_callback(std::function&)> f) { + cell_load_callback_ = std::move(f); + } + private: struct Node { bool is_loaded{false}; @@ -68,8 +76,9 @@ class CellUsageTree : public std::enable_shared_from_this { }; bool use_mark_{false}; std::vector nodes_{2}; + std::function&)> cell_load_callback_; - void on_load(NodeId node_id); + void on_load(NodeId node_id, const td::Ref& cell); NodeId create_node(NodeId parent); }; } // namespace vm diff --git a/crypto/vm/cells/MerkleProof.h b/crypto/vm/cells/MerkleProof.h index 9c50fd078..fc2cb6ebd 100644 --- a/crypto/vm/cells/MerkleProof.h +++ b/crypto/vm/cells/MerkleProof.h @@ -66,6 +66,10 @@ class MerkleProofBuilder { td::Result> extract_proof() const; bool extract_proof_to(Ref &proof_root) const; td::Result extract_proof_boc() const; + + void set_cell_load_callback(std::function&)> f) { + usage_tree->set_cell_load_callback(std::move(f)); + } }; } // namespace vm diff --git a/crypto/vm/cells/UsageCell.h b/crypto/vm/cells/UsageCell.h index bf15bb56f..3e6e88982 100644 --- a/crypto/vm/cells/UsageCell.h +++ b/crypto/vm/cells/UsageCell.h @@ -39,7 +39,7 @@ class UsageCell : public Cell { // load interface td::Result load_cell() const override { TRY_RESULT(loaded_cell, cell_->load_cell()); - if (tree_node_.on_load()) { + if (tree_node_.on_load(loaded_cell.data_cell)) { CHECK(loaded_cell.tree_node.empty()); loaded_cell.tree_node = tree_node_; } diff --git a/keys/keys.hpp b/keys/keys.hpp index 72d0845ac..2e517b2e4 100644 --- a/keys/keys.hpp +++ b/keys/keys.hpp @@ -260,6 +260,10 @@ class PublicKey { td::BufferSlice export_as_slice() const; static td::Result import(td::Slice s); + bool is_ed25519() const { + return pub_key_.get_offset() == pub_key_.offset(); + } + pubkeys::Ed25519 ed25519_value() const { CHECK(pub_key_.get_offset() == pub_key_.offset()); return pub_key_.get(); diff --git a/overlay/overlay-broadcast.cpp b/overlay/overlay-broadcast.cpp index 03991b76b..615b3e7c8 100644 --- a/overlay/overlay-broadcast.cpp +++ b/overlay/overlay-broadcast.cpp @@ -68,7 +68,7 @@ td::Status BroadcastSimple::run_checks() { td::Status BroadcastSimple::distribute() { auto B = serialize(); - auto nodes = overlay_->get_neighbours(3); + auto nodes = overlay_->get_neighbours(overlay_->propagate_broadcast_to()); auto manager = overlay_->overlay_manager(); for (auto &n : nodes) { @@ -115,7 +115,8 @@ td::Status BroadcastSimple::run() { return run_continue(); } -td::Status BroadcastSimple::create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, tl_object_ptr broadcast) { +td::Status BroadcastSimple::create(OverlayImpl *overlay, adnl::AdnlNodeIdShort src_peer_id, + tl_object_ptr broadcast) { auto src = PublicKey{broadcast->src_}; auto data_hash = sha256_bits256(broadcast->data_.as_slice()); auto broadcast_hash = compute_broadcast_id(src, data_hash, broadcast->flags_); diff --git a/overlay/overlay-fec-broadcast.cpp b/overlay/overlay-fec-broadcast.cpp index cd030742a..5a0ad10dd 100644 --- a/overlay/overlay-fec-broadcast.cpp +++ b/overlay/overlay-fec-broadcast.cpp @@ -112,7 +112,7 @@ td::Status BroadcastFec::distribute_part(td::uint32 seqno) { td::BufferSlice data_short = std::move(tls.first); td::BufferSlice data = std::move(tls.second); - auto nodes = overlay_->get_neighbours(5); + auto nodes = overlay_->get_neighbours(overlay_->propagate_broadcast_to()); auto manager = overlay_->overlay_manager(); for (auto &n : nodes) { diff --git a/overlay/overlay-id.hpp b/overlay/overlay-id.hpp index c22b0faa7..2625773bd 100644 --- a/overlay/overlay-id.hpp +++ b/overlay/overlay-id.hpp @@ -21,8 +21,14 @@ #include "auto/tl/ton_api.h" #include "adnl/adnl-node-id.hpp" #include "overlay/overlays.h" +#include "td/utils/SharedSlice.h" +#include "td/utils/buffer.h" #include "td/utils/overloaded.h" #include "keys/encryptor.h" +#include "td/utils/port/StdStreams.h" +#include "td/utils/unique_ptr.h" +#include +#include namespace ton { @@ -30,18 +36,30 @@ namespace overlay { class OverlayNode { public: - explicit OverlayNode(adnl::AdnlNodeIdShort self_id, OverlayIdShort overlay) { + explicit OverlayNode(adnl::AdnlNodeIdShort self_id, OverlayIdShort overlay, td::uint32 flags) { source_ = self_id; overlay_ = overlay; + flags_ = flags; version_ = static_cast(td::Clocks::system()); } static td::Result create(const tl_object_ptr &node) { TRY_RESULT(source, adnl::AdnlNodeIdFull::create(node->id_)); - return OverlayNode{source, OverlayIdShort{node->overlay_}, node->version_, node->signature_.as_slice()}; + return OverlayNode{source, OverlayIdShort{node->overlay_}, 0, node->version_, node->signature_.as_slice()}; } - OverlayNode(td::Variant source, OverlayIdShort overlay, + static td::Result create(const tl_object_ptr &node) { + TRY_RESULT(source, adnl::AdnlNodeIdFull::create(node->id_)); + auto res = OverlayNode{source, OverlayIdShort{node->overlay_}, (td::uint32)node->flags_, node->version_, + node->signature_.as_slice()}; + res.update_certificate(OverlayMemberCertificate(node->certificate_.get())); + return res; + } + OverlayNode(td::Variant source, OverlayIdShort overlay, td::uint32 flags, td::int32 version, td::Slice signature) - : source_(std::move(source)), overlay_(overlay), version_(version), signature_(td::SharedSlice(signature)) { + : source_(std::move(source)) + , overlay_(overlay) + , flags_(flags) + , version_(version) + , signature_(td::SharedSlice(signature)) { } OverlayNode(td::Variant source, OverlayIdShort overlay, td::int32 version, td::SharedSlice signature) @@ -64,10 +82,17 @@ class OverlayNode { } td::BufferSlice to_sign() const { - auto obj = create_tl_object(nullptr, overlay_.tl(), version_); - source_.visit(td::overloaded([&](const adnl::AdnlNodeIdShort &id) { obj->id_ = id.tl(); }, - [&](const adnl::AdnlNodeIdFull &id) { obj->id_ = id.compute_short_id().tl(); })); - return serialize_tl_object(obj, true); + if (flags_ == 0) { + auto obj = create_tl_object(nullptr, overlay_.tl(), version_); + source_.visit(td::overloaded([&](const adnl::AdnlNodeIdShort &id) { obj->id_ = id.tl(); }, + [&](const adnl::AdnlNodeIdFull &id) { obj->id_ = id.compute_short_id().tl(); })); + return serialize_tl_object(obj, true); + } else { + auto obj = create_tl_object(nullptr, overlay_.tl(), flags_, version_); + source_.visit(td::overloaded([&](const adnl::AdnlNodeIdShort &id) { obj->id_ = id.tl(); }, + [&](const adnl::AdnlNodeIdFull &id) { obj->id_ = id.compute_short_id().tl(); })); + return serialize_tl_object(obj, true); + } } void update_adnl_id(adnl::AdnlNodeIdFull node_id) { source_ = node_id; @@ -81,6 +106,9 @@ class OverlayNode { td::int32 version() const { return version_; } + td::uint32 flags() const { + return flags_; + } td::BufferSlice signature() const { return signature_.clone_as_buffer_slice(); } @@ -103,15 +131,69 @@ class OverlayNode { [&](const adnl::AdnlNodeIdFull &id) { obj->id_ = id.tl(); })); return obj; } + tl_object_ptr tl_v2() const { + tl_object_ptr cert; + if (cert_ && !cert_->empty()) { + cert = cert_->tl(); + } else { + cert = create_tl_object(); + } + auto obj = create_tl_object(nullptr, overlay_.tl(), flags_, version_, + signature_.clone_as_buffer_slice(), std::move(cert)); + source_.visit(td::overloaded([&](const adnl::AdnlNodeIdShort &id) { UNREACHABLE(); }, + [&](const adnl::AdnlNodeIdFull &id) { obj->id_ = id.tl(); })); + return obj; + } OverlayNode clone() const { - return OverlayNode{source_, overlay_, version_, signature_.clone()}; + auto res = OverlayNode{source_, overlay_, version_, signature_.clone()}; + if (cert_) { + res.cert_ = td::make_unique(*cert_); + } + return res; + } + + const OverlayMemberCertificate *certificate() const { + if (cert_) { + return cert_.get(); + } + return &empty_certificate_; + } + + void update_certificate(OverlayMemberCertificate cert) { + if (!cert_ || cert_->empty() || cert_->is_expired() || cert.is_newer(*cert_)) { + cert_ = td::make_unique(std::move(cert)); + } + } + + void update(OverlayNode from) { + if (version_ < from.version_) { + source_ = from.source_; + overlay_ = from.overlay_; + flags_ = from.flags_; + version_ = from.version_; + signature_ = from.signature_.clone(); + } + if (from.cert_ && !from.cert_->empty()) { + update_certificate(std::move(*from.cert_)); + } + } + + void clear_certificate() { + cert_ = nullptr; + } + + bool has_full_id() const { + return source_.get_offset() == source_.offset(); } private: td::Variant source_; OverlayIdShort overlay_; + td::uint32 flags_; td::int32 version_; + td::unique_ptr cert_; td::SharedSlice signature_; + static const OverlayMemberCertificate empty_certificate_; }; } // namespace overlay diff --git a/overlay/overlay-manager.cpp b/overlay/overlay-manager.cpp index b9eb95b98..878f6637a 100644 --- a/overlay/overlay-manager.cpp +++ b/overlay/overlay-manager.cpp @@ -18,6 +18,7 @@ */ #include "overlay-manager.h" #include "auto/tl/ton_api.h" +#include "auto/tl/ton_api.hpp" #include "overlay.h" #include "adnl/utils.hpp" @@ -28,9 +29,9 @@ #include "td/db/RocksDb.h" #include "td/utils/Status.h" +#include "td/utils/buffer.h" #include "td/utils/overloaded.h" -#include "keys/encryptor.h" #include "td/utils/port/Poll.h" #include @@ -42,13 +43,13 @@ void OverlayManager::update_dht_node(td::actor::ActorId dht) { dht_node_ = dht; for (auto &X : overlays_) { for (auto &Y : X.second) { - td::actor::send_closure(Y.second, &Overlay::update_dht_node, dht); + td::actor::send_closure(Y.second.overlay, &Overlay::update_dht_node, dht); } } } void OverlayManager::register_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, - td::actor::ActorOwn overlay) { + OverlayMemberCertificate cert, td::actor::ActorOwn overlay) { auto it = overlays_.find(local_id); VLOG(OVERLAY_INFO) << this << ": registering overlay " << overlay_id << "@" << local_id; if (it == overlays_.end()) { @@ -58,19 +59,34 @@ void OverlayManager::register_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdS td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id, adnl::Adnl::int_to_bytestring(ton_api::overlay_query::ID), std::make_unique(actor_id(this))); + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id, + adnl::Adnl::int_to_bytestring(ton_api::overlay_messageWithExtra::ID), + std::make_unique(actor_id(this))); + td::actor::send_closure(adnl_, &adnl::Adnl::subscribe, local_id, + adnl::Adnl::int_to_bytestring(ton_api::overlay_queryWithExtra::ID), + std::make_unique(actor_id(this))); } - overlays_[local_id][overlay_id] = std::move(overlay); - - auto P = td::PromiseCreator::lambda([id = overlays_[local_id][overlay_id].get()](td::Result R) { - R.ensure(); - auto value = R.move_as_ok(); - if (value.status == td::KeyValue::GetStatus::Ok) { - auto F = fetch_tl_object(std::move(value.value), true); - F.ensure(); - auto nodes = std::move(F.move_as_ok()->nodes_); - td::actor::send_closure(id, &Overlay::receive_nodes_from_db, std::move(nodes)); - } - }); + overlays_[local_id][overlay_id] = OverlayDescription{std::move(overlay), std::move(cert)}; + + auto P = + td::PromiseCreator::lambda([id = overlays_[local_id][overlay_id].overlay.get()](td::Result R) { + R.ensure(); + auto value = R.move_as_ok(); + if (value.status == td::KeyValue::GetStatus::Ok) { + auto F = fetch_tl_object(std::move(value.value), true); + F.ensure(); + ton_api::downcast_call( + *F.move_as_ok(), td::overloaded( + [&](ton_api::overlay_db_nodes &V) { + auto nodes = std::move(V.nodes_); + td::actor::send_closure(id, &Overlay::receive_nodes_from_db, std::move(nodes)); + }, + [&](ton_api::overlay_db_nodesV2 &V) { + auto nodes = std::move(V.nodes_); + td::actor::send_closure(id, &Overlay::receive_nodes_from_db_v2, std::move(nodes)); + })); + } + }); auto key = create_hash_tl_object(local_id.bits256_value(), overlay_id.bits256_value()); db_.get(key, std::move(P)); } @@ -84,6 +100,10 @@ void OverlayManager::delete_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdSho adnl::Adnl::int_to_bytestring(ton_api::overlay_message::ID)); td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id, adnl::Adnl::int_to_bytestring(ton_api::overlay_query::ID)); + td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id, + adnl::Adnl::int_to_bytestring(ton_api::overlay_messageWithExtra::ID)); + td::actor::send_closure(adnl_, &adnl::Adnl::unsubscribe, local_id, + adnl::Adnl::int_to_bytestring(ton_api::overlay_queryWithExtra::ID)); overlays_.erase(it); } } @@ -101,74 +121,113 @@ void OverlayManager::create_public_overlay_ex(adnl::AdnlNodeIdShort local_id, Ov td::string scope, OverlayOptions opts) { CHECK(!dht_node_.empty()); auto id = overlay_id.compute_short_id(); - register_overlay(local_id, id, - Overlay::create(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id), - std::move(callback), std::move(rules), scope, std::move(opts))); + register_overlay(local_id, id, OverlayMemberCertificate{}, + Overlay::create_public(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id), + std::move(callback), std::move(rules), scope, std::move(opts))); } void OverlayManager::create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, OverlayPrivacyRules rules, std::string scope) { + create_private_overlay_ex(local_id, std::move(overlay_id), std::move(nodes), std::move(callback), std::move(rules), + std::move(scope), {}); +} + +void OverlayManager::create_private_overlay_ex(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + std::vector nodes, + std::unique_ptr callback, OverlayPrivacyRules rules, + std::string scope, OverlayOptions opts) { + auto id = overlay_id.compute_short_id(); + register_overlay(local_id, id, OverlayMemberCertificate{}, + Overlay::create_private(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id), + std::move(nodes), std::move(callback), std::move(rules), std::move(scope), + std::move(opts))); +} + +void OverlayManager::create_semiprivate_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + std::vector nodes, + std::vector root_public_keys, + OverlayMemberCertificate certificate, + std::unique_ptr callback, OverlayPrivacyRules rules, + td::string scope, OverlayOptions opts) { auto id = overlay_id.compute_short_id(); - register_overlay(local_id, id, - Overlay::create(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id), - std::move(nodes), std::move(callback), std::move(rules), std::move(scope))); + register_overlay( + local_id, id, certificate, + Overlay::create_semiprivate(keyring_, adnl_, actor_id(this), dht_node_, local_id, std::move(overlay_id), + std::move(nodes), std::move(root_public_keys), certificate, std::move(callback), + std::move(rules), std::move(scope), std::move(opts))); } void OverlayManager::receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data) { - auto R = fetch_tl_prefix(data, true); - - if (R.is_error()) { - VLOG(OVERLAY_WARNING) << this << ": can not parse overlay message: " << R.move_as_error(); - return; + OverlayIdShort overlay_id; + tl_object_ptr extra; + auto R = fetch_tl_prefix(data, true); + if (R.is_ok()) { + overlay_id = OverlayIdShort{R.ok()->overlay_}; + extra = std::move(R.ok()->extra_); + } else { + auto R2 = fetch_tl_prefix(data, true); + if (R2.is_ok()) { + overlay_id = OverlayIdShort{R2.ok()->overlay_}; + } else { + VLOG(OVERLAY_WARNING) << this << ": can not parse overlay message [" << src << "->" << dst + << "]: " << R2.move_as_error(); + return; + } } - auto M = R.move_as_ok(); - auto it = overlays_.find(dst); if (it == overlays_.end()) { - VLOG(OVERLAY_NOTICE) << this << ": message to unknown overlay " << M->overlay_ << "@" << dst; + VLOG(OVERLAY_NOTICE) << this << ": message to unknown overlay " << overlay_id << "@" << dst; return; } - auto it2 = it->second.find(OverlayIdShort{M->overlay_}); + auto it2 = it->second.find(overlay_id); if (it2 == it->second.end()) { - VLOG(OVERLAY_NOTICE) << this << ": message to localid is not in overlay " << M->overlay_ << "@" << dst; + VLOG(OVERLAY_NOTICE) << this << ": message to localid is not in overlay " << overlay_id << "@" << dst; return; } - td::actor::send_closure(it2->second, &Overlay::update_throughput_in_ctr, src, (td::uint32)data.size(), false); - td::actor::send_closure(it2->second, &Overlay::receive_message, src, std::move(data)); + td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_in_ctr, src, (td::uint32)data.size(), false); + td::actor::send_closure(it2->second.overlay, &Overlay::receive_message, src, std::move(extra), std::move(data)); } void OverlayManager::receive_query(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) { - auto R = fetch_tl_prefix(data, true); - - if (R.is_error()) { - VLOG(OVERLAY_WARNING) << this << ": can not parse overlay query [" << src << "->" << dst - << "]: " << R.move_as_error(); - promise.set_error(td::Status::Error(ErrorCode::protoviolation, "bad overlay query header")); - return; + OverlayIdShort overlay_id; + tl_object_ptr extra; + auto R = fetch_tl_prefix(data, true); + if (R.is_ok()) { + overlay_id = OverlayIdShort{R.ok()->overlay_}; + extra = std::move(R.ok()->extra_); + } else { + auto R2 = fetch_tl_prefix(data, true); + if (R2.is_ok()) { + overlay_id = OverlayIdShort{R2.ok()->overlay_}; + } else { + VLOG(OVERLAY_WARNING) << this << ": can not parse overlay query [" << src << "->" << dst + << "]: " << R2.move_as_error(); + promise.set_error(td::Status::Error(ErrorCode::protoviolation, "bad overlay query header")); + return; + } } - auto M = R.move_as_ok(); - auto it = overlays_.find(dst); if (it == overlays_.end()) { - VLOG(OVERLAY_NOTICE) << this << ": query to unknown overlay " << M->overlay_ << "@" << dst << " from " << src; + VLOG(OVERLAY_NOTICE) << this << ": query to unknown overlay " << overlay_id << "@" << dst << " from " << src; promise.set_error(td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad local_id " << dst)); return; } - auto it2 = it->second.find(OverlayIdShort{M->overlay_}); + auto it2 = it->second.find(overlay_id); if (it2 == it->second.end()) { - VLOG(OVERLAY_NOTICE) << this << ": query to localid not in overlay " << M->overlay_ << "@" << dst << " from " << src; - promise.set_error(td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad overlay_id " << M->overlay_)); + VLOG(OVERLAY_NOTICE) << this << ": query to localid not in overlay " << overlay_id << "@" << dst << " from " << src; + promise.set_error(td::Status::Error(ErrorCode::protoviolation, PSTRING() << "bad overlay_id " << overlay_id)); return; } - td::actor::send_closure(it2->second, &Overlay::update_throughput_in_ctr, src, (td::uint32)data.size(), true); - td::actor::send_closure(it2->second, &Overlay::receive_query, src, std::move(data), std::move(promise)); + td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_in_ctr, src, (td::uint32)data.size(), true); + td::actor::send_closure(it2->second.overlay, &Overlay::receive_query, src, std::move(extra), std::move(data), + std::move(promise)); } void OverlayManager::send_query_via(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdShort src, OverlayIdShort overlay_id, @@ -176,35 +235,64 @@ void OverlayManager::send_query_via(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdS td::BufferSlice query, td::uint64 max_answer_size, td::actor::ActorId via) { CHECK(query.size() <= adnl::Adnl::huge_packet_max_size()); - + + auto extra = create_tl_object(); + extra->flags_ = 0; + auto it = overlays_.find(src); if (it != overlays_.end()) { auto it2 = it->second.find(overlay_id); if (it2 != it->second.end()) { - td::actor::send_closure(it2->second, &Overlay::update_throughput_out_ctr, dst, (td::uint32)query.size(), true); + td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_out_ctr, dst, (td::uint32)query.size(), + true); + if (!it2->second.member_certificate.empty()) { + extra->flags_ |= 1; + extra->certificate_ = it2->second.member_certificate.tl(); + } } } - - td::actor::send_closure( - via, &adnl::AdnlSenderInterface::send_query_ex, src, dst, std::move(name), std::move(promise), timeout, - create_serialize_tl_object_suffix(query.as_slice(), overlay_id.tl()), max_answer_size); + + auto extra_flags = extra->flags_; + td::BufferSlice serialized_query = + (extra_flags ? create_serialize_tl_object_suffix( + query.as_slice(), overlay_id.tl(), std::move(extra)) + : create_serialize_tl_object_suffix(query.as_slice(), overlay_id.tl())); + + td::actor::send_closure(via, &adnl::AdnlSenderInterface::send_query_ex, src, dst, std::move(name), std::move(promise), + timeout, std::move(serialized_query), max_answer_size); } void OverlayManager::send_message_via(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdShort src, OverlayIdShort overlay_id, td::BufferSlice object, td::actor::ActorId via) { CHECK(object.size() <= adnl::Adnl::huge_packet_max_size()); - + + auto extra = create_tl_object(); + extra->flags_ = 0; + auto it = overlays_.find(src); if (it != overlays_.end()) { auto it2 = it->second.find(overlay_id); if (it2 != it->second.end()) { - td::actor::send_closure(it2->second, &Overlay::update_throughput_out_ctr, dst, (td::uint32)object.size(), false); + td::actor::send_closure(it2->second.overlay, &Overlay::update_throughput_out_ctr, dst, (td::uint32)object.size(), + false); + if (!it2->second.member_certificate.empty()) { + // do not send certificate here, we hope that all our neighbours already know of out certificate + // we send it every second to some random nodes. Here we don't want to increase the size of the message + if (false) { + extra->flags_ |= 1; + extra->certificate_ = it2->second.member_certificate.tl(); + } + } } } - - td::actor::send_closure( - via, &adnl::AdnlSenderInterface::send_message, src, dst, - create_serialize_tl_object_suffix(object.as_slice(), overlay_id.tl())); + + auto extra_flags = extra->flags_; + td::BufferSlice serialized_message = + (extra_flags ? create_serialize_tl_object_suffix( + object.as_slice(), overlay_id.tl(), std::move(extra)) + : create_serialize_tl_object_suffix(object.as_slice(), overlay_id.tl())); + + td::actor::send_closure(via, &adnl::AdnlSenderInterface::send_message, src, dst, std::move(serialized_message)); } void OverlayManager::send_broadcast(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, td::BufferSlice object) { @@ -218,7 +306,7 @@ void OverlayManager::send_broadcast_ex(adnl::AdnlNodeIdShort local_id, OverlayId if (it != overlays_.end()) { auto it2 = it->second.find(overlay_id); if (it2 != it->second.end()) { - td::actor::send_closure(it2->second, &Overlay::send_broadcast, send_as, flags, std::move(object)); + td::actor::send_closure(it2->second.overlay, &Overlay::send_broadcast, send_as, flags, std::move(object)); } } } @@ -235,7 +323,7 @@ void OverlayManager::send_broadcast_fec_ex(adnl::AdnlNodeIdShort local_id, Overl if (it != overlays_.end()) { auto it2 = it->second.find(overlay_id); if (it2 != it->second.end()) { - td::actor::send_closure(it2->second, &Overlay::send_broadcast_fec, send_as, flags, std::move(object)); + td::actor::send_closure(it2->second.overlay, &Overlay::send_broadcast_fec, send_as, flags, std::move(object)); } } } @@ -246,7 +334,7 @@ void OverlayManager::set_privacy_rules(adnl::AdnlNodeIdShort local_id, OverlayId if (it != overlays_.end()) { auto it2 = it->second.find(overlay_id); if (it2 != it->second.end()) { - td::actor::send_closure(it2->second, &Overlay::set_privacy_rules, std::move(rules)); + td::actor::send_closure(it2->second.overlay, &Overlay::set_privacy_rules, std::move(rules)); } } } @@ -257,23 +345,54 @@ void OverlayManager::update_certificate(adnl::AdnlNodeIdShort local_id, OverlayI if (it != overlays_.end()) { auto it2 = it->second.find(overlay_id); if (it2 != it->second.end()) { - td::actor::send_closure(it2->second, &Overlay::add_certificate, key, std::move(cert)); + td::actor::send_closure(it2->second.overlay, &Overlay::add_certificate, key, std::move(cert)); } } } -void OverlayManager::get_overlay_random_peers(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, - td::uint32 max_peers, - td::Promise> promise) { +void OverlayManager::update_member_certificate(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, + OverlayMemberCertificate certificate) { auto it = overlays_.find(local_id); if (it != overlays_.end()) { auto it2 = it->second.find(overlay_id); if (it2 != it->second.end()) { - td::actor::send_closure(it2->second, &Overlay::get_overlay_random_peers, max_peers, std::move(promise)); + it2->second.member_certificate = certificate; + td::actor::send_closure(it2->second.overlay, &Overlay::update_member_certificate, certificate); } } } +void OverlayManager::update_root_member_list(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, + std::vector nodes, + std::vector root_public_keys, + OverlayMemberCertificate certificate) { + auto it = overlays_.find(local_id); + if (it != overlays_.end()) { + auto it2 = it->second.find(overlay_id); + if (it2 != it->second.end()) { + it2->second.member_certificate = certificate; + td::actor::send_closure(it2->second.overlay, &Overlay::update_root_member_list, std::move(nodes), + std::move(root_public_keys), std::move(certificate)); + } + } +} + +void OverlayManager::get_overlay_random_peers(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, + td::uint32 max_peers, + td::Promise> promise) { + auto it = overlays_.find(local_id); + if (it == overlays_.end()) { + promise.set_error(td::Status::Error(PSTRING() << "no such local id " << local_id)); + return; + } + auto it2 = it->second.find(overlay_id); + if (it2 == it->second.end()) { + promise.set_error(td::Status::Error(PSTRING() << "no such overlay " << overlay_id)); + return; + } + td::actor::send_closure(it2->second.overlay, &Overlay::get_overlay_random_peers, max_peers, std::move(promise)); +} + td::actor::ActorOwn Overlays::create(std::string db_root, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId dht) { return td::actor::create_actor("overlaymanager", db_root, keyring, adnl, dht); @@ -334,7 +453,7 @@ void OverlayManager::get_stats(td::Promise> R) { if (R.is_ok()) { td::actor::send_closure(act, &Cb::receive_answer, R.move_as_ok()); @@ -348,6 +467,19 @@ void OverlayManager::get_stats(td::Promisesecond.find(overlay); + if (it2 == it->second.end()) { + return; + } + td::actor::send_closure(it2->second.overlay, &Overlay::forget_peer, peer_id); +} + Certificate::Certificate(PublicKey issued_by, td::int32 expire_at, td::uint32 max_size, td::uint32 flags, td::BufferSlice signature) : issued_by_(issued_by) @@ -454,6 +586,35 @@ tl_object_ptr Certificate::empty_tl() { return create_tl_object(); } +OverlayMemberCertificate::OverlayMemberCertificate(const ton_api::overlay_MemberCertificate *cert) { + if (!cert) { + expire_at_ = std::numeric_limits::max(); + return; + } + if (cert->get_id() == ton_api::overlay_emptyMemberCertificate::ID) { + expire_at_ = std::numeric_limits::max(); + return; + } + CHECK(cert->get_id() == ton_api::overlay_memberCertificate::ID); + const auto *real_cert = static_cast(cert); + signed_by_ = PublicKey(real_cert->issued_by_); + flags_ = real_cert->flags_; + slot_ = real_cert->slot_; + expire_at_ = real_cert->expire_at_; + signature_ = td::SharedSlice(real_cert->signature_.as_slice()); +} + +td::Status OverlayMemberCertificate::check_signature(const adnl::AdnlNodeIdShort &node) { + if (is_expired()) { + return td::Status::Error(ErrorCode::notready, "certificate is expired"); + } + td::BufferSlice data_to_sign = to_sign_data(node); + + TRY_RESULT(encryptor, signed_by_.create_encryptor()); + TRY_STATUS(encryptor->check_signature(data_to_sign.as_slice(), signature_.as_slice())); + return td::Status::OK(); +} + } // namespace overlay } // namespace ton diff --git a/overlay/overlay-manager.h b/overlay/overlay-manager.h index 1b9c75a4e..12206e048 100644 --- a/overlay/overlay-manager.h +++ b/overlay/overlay-manager.h @@ -53,11 +53,19 @@ class OverlayManager : public Overlays { void create_public_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope) override; void create_public_overlay_ex(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, - std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope, - OverlayOptions opts) override; + std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope, + OverlayOptions opts) override; + void create_semiprivate_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + std::vector nodes, std::vector root_public_keys, + OverlayMemberCertificate certificate, + std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope, + OverlayOptions opts) override; void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, OverlayPrivacyRules rules, std::string scope) override; + void create_private_overlay_ex(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + std::vector nodes, std::unique_ptr callback, + OverlayPrivacyRules rules, std::string scope, OverlayOptions opts) override; void delete_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id) override; void send_query(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdShort src, OverlayIdShort overlay_id, std::string name, td::Promise promise, td::Timestamp timeout, td::BufferSlice query) override { @@ -84,6 +92,11 @@ class OverlayManager : public Overlays { void set_privacy_rules(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, OverlayPrivacyRules rules) override; void update_certificate(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, PublicKeyHash key, std::shared_ptr cert) override; + void update_member_certificate(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, + OverlayMemberCertificate certificate) override; + void update_root_member_list(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, + std::vector nodes, std::vector root_public_keys, + OverlayMemberCertificate certificate) override; void get_overlay_random_peers(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, td::uint32 max_peers, td::Promise> promise) override; @@ -92,10 +105,12 @@ class OverlayManager : public Overlays { td::Promise promise); void receive_message(adnl::AdnlNodeIdShort src, adnl::AdnlNodeIdShort dst, td::BufferSlice data); - void register_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, + void register_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, OverlayMemberCertificate cert, td::actor::ActorOwn overlay); void get_stats(td::Promise> promise) override; + void forget_peer(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, adnl::AdnlNodeIdShort peer_id) override; + struct PrintId {}; PrintId print_id() const { @@ -103,7 +118,11 @@ class OverlayManager : public Overlays { } private: - std::map>> overlays_; + struct OverlayDescription { + td::actor::ActorOwn overlay; + OverlayMemberCertificate member_certificate; + }; + std::map> overlays_; std::string db_root_; diff --git a/overlay/overlay-peers.cpp b/overlay/overlay-peers.cpp index e81fecc70..d37002444 100644 --- a/overlay/overlay-peers.cpp +++ b/overlay/overlay-peers.cpp @@ -16,89 +16,165 @@ Copyright 2017-2020 Telegram Systems LLP */ +#include "adnl/adnl-node-id.hpp" +#include "adnl/adnl-node.h" +#include "auto/tl/ton_api.h" #include "overlay.hpp" +#include "td/utils/Status.h" +#include "td/utils/Time.h" +#include "td/utils/port/signals.h" +#include +#include namespace ton { namespace overlay { -void OverlayImpl::del_peer(adnl::AdnlNodeIdShort id) { - auto P = peers_.get(id); - CHECK(P != nullptr); +void OverlayImpl::del_peer(const adnl::AdnlNodeIdShort &id) { + auto P = peer_list_.peers_.get(id); + if (P == nullptr) { + return; + } + if (P->is_permanent_member()) { + VLOG(OVERLAY_DEBUG) << this << ": not deleting peer " << id << ": a permanent member"; + return; + } VLOG(OVERLAY_DEBUG) << this << ": deleting peer " << id; if (P->is_neighbour()) { - VLOG(OVERLAY_INFO) << this << ": deleting neighbour " << id; - bool deleted = false; - for (auto &n : neighbours_) { - if (n == id) { - n = neighbours_[neighbours_.size() - 1]; - neighbours_.resize(neighbours_.size() - 1); - deleted = true; - break; - } + del_from_neighbour_list(P); + } + peer_list_.peers_.remove(id); + peer_list_.bad_peers_.erase(id); +} + +void OverlayImpl::del_from_neighbour_list(OverlayPeer *P) { + CHECK(P); + if (!P->is_neighbour()) { + return; + } + auto id = P->get_id(); + bool deleted = false; + auto &neighbours = peer_list_.neighbours_; + for (auto &n : neighbours) { + if (n == id) { + n = neighbours[neighbours.size() - 1]; + neighbours.resize(neighbours.size() - 1); + deleted = true; + break; } - CHECK(deleted); - P->set_neighbour(false); } - peers_.remove(id); - bad_peers_.erase(id); - update_neighbours(0); + CHECK(deleted); + P->set_neighbour(false); +} + +void OverlayImpl::del_from_neighbour_list(const adnl::AdnlNodeIdShort &id) { + auto P = peer_list_.peers_.get(id); + CHECK(P != nullptr); + return del_from_neighbour_list(P); } void OverlayImpl::del_some_peers() { - if (!public_) { + if (overlay_type_ == OverlayType::FixedMemberList) { return; } - while (peers_.size() > max_peers()) { + const size_t max_iterations = 10; + size_t iteration_seqno = 0; + while (peer_list_.peers_.size() > max_peers() && iteration_seqno++ < max_iterations) { OverlayPeer *P; - if (bad_peers_.empty()) { + if (peer_list_.bad_peers_.empty()) { P = get_random_peer(); } else { - auto it = bad_peers_.upper_bound(next_bad_peer_); - if (it == bad_peers_.end()) { - it = bad_peers_.begin(); + auto it = peer_list_.bad_peers_.upper_bound(peer_list_.next_bad_peer_); + if (it == peer_list_.bad_peers_.end()) { + it = peer_list_.bad_peers_.begin(); } - P = peers_.get(next_bad_peer_ = *it); + P = peer_list_.peers_.get(peer_list_.next_bad_peer_ = *it); } - if (P) { + if (P && !P->is_permanent_member()) { auto id = P->get_id(); del_peer(id); } } + update_neighbours(0); } -void OverlayImpl::do_add_peer(OverlayNode node) { - auto id = node.adnl_id_short(); - - auto V = peers_.get(id); - if (V) { - VLOG(OVERLAY_DEBUG) << this << ": updating peer " << id << " up to version " << node.version(); - V->update(std::move(node)); - } else { - VLOG(OVERLAY_DEBUG) << this << ": adding peer " << id << " of version " << node.version(); - peers_.insert(id, OverlayPeer(std::move(node))); - - del_some_peers(); - update_neighbours(0); +td::Status OverlayImpl::validate_peer_certificate(const adnl::AdnlNodeIdShort &node, + const OverlayMemberCertificate &cert) { + if (cert.empty()) { + if (is_persistent_node(node) || overlay_type_ == OverlayType::Public) { + return td::Status::OK(); + } + return td::Status::Error(ErrorCode::protoviolation, "no member certificate found"); + } + if (cert.is_expired()) { + return td::Status::Error(ErrorCode::timeout, "member certificate is expired"); + } + if (cert.slot() < 0 || cert.slot() >= opts_.max_slaves_in_semiprivate_overlay_) { + return td::Status::Error(ErrorCode::timeout, "member certificate has invalid slot"); + } + const auto &issued_by = cert.issued_by(); + auto it = peer_list_.root_public_keys_.find(issued_by.compute_short_id()); + if (it == peer_list_.root_public_keys_.end()) { + return td::Status::Error(ErrorCode::protoviolation, "member certificate is signed by unknown public key"); + } + if (it->second.size() > (size_t)cert.slot()) { + auto &el = it->second[cert.slot()]; + if (cert.expire_at() < el.expire_at) { + return td::Status::Error(ErrorCode::protoviolation, + "member certificate rejected, because we know of newer certificate at the same slot"); + } else if (cert.expire_at() == el.expire_at) { + if (node < el.node) { + return td::Status::Error(ErrorCode::protoviolation, + "member certificate rejected, because we know of newer certificate at the same slot"); + } else if (el.node == node) { + // we could return OK here, but we must make sure, that the unchecked signature will not be used for updating PeerNode. + } + } + } + auto R = get_encryptor(issued_by); + if (R.is_error()) { + return R.move_as_error_prefix("failed to check member certificate: failed to create encryptor: "); } + auto enc = R.move_as_ok(); + auto S = enc->check_signature(cert.to_sign_data(node).as_slice(), cert.signature()); + if (S.is_error()) { + return S.move_as_error_prefix("failed to check member certificate: bad signature: "); + } + if (it->second.size() <= (size_t)cert.slot()) { + it->second.resize((size_t)cert.slot() + 1); + } + it->second[cert.slot()].expire_at = cert.expire_at(); + it->second[cert.slot()].node = node; + return td::Status::OK(); } -void OverlayImpl::add_peer_in_cont(OverlayNode node) { - CHECK(public_); +td::Status OverlayImpl::validate_peer_certificate(const adnl::AdnlNodeIdShort &node, + ton_api::overlay_MemberCertificate *cert) { + OverlayMemberCertificate ncert(cert); + return validate_peer_certificate(node, ncert); +} - do_add_peer(std::move(node)); +td::Status OverlayImpl::validate_peer_certificate(const adnl::AdnlNodeIdShort &node, + const OverlayMemberCertificate *cert) { + if (!cert) { + if (is_persistent_node(node) || overlay_type_ == OverlayType::Public) { + return td::Status::OK(); + } + return td::Status::Error(ErrorCode::protoviolation, "no member certificate found"); + } + return validate_peer_certificate(node, *cert); } -void OverlayImpl::add_peer_in(OverlayNode node) { - CHECK(public_); +void OverlayImpl::add_peer(OverlayNode node) { + CHECK(overlay_type_ != OverlayType::FixedMemberList); if (node.overlay_id() != overlay_id_) { VLOG(OVERLAY_WARNING) << this << ": received node with bad overlay"; return; } auto t = td::Clocks::system(); - if (node.version() + 600 < t || node.version() > t + 60) { + if (node.version() + Overlays::overlay_peer_ttl() < t || node.version() > t + 60) { VLOG(OVERLAY_INFO) << this << ": ignoring node of too old version " << node.version(); return; } @@ -115,35 +191,80 @@ void OverlayImpl::add_peer_in(OverlayNode node) { return; } - add_peer_in_cont(std::move(node)); + if (overlay_type_ == OverlayType::CertificatedMembers) { + auto R = validate_peer_certificate(node.adnl_id_short(), *node.certificate()); + if (R.is_error()) { + VLOG(OVERLAY_WARNING) << this << ": bad peer certificate node=" << node.adnl_id_short() << ": " + << R.move_as_error(); + UNREACHABLE(); + return; + } + } + + auto id = node.adnl_id_short(); + + auto V = peer_list_.peers_.get(id); + if (V) { + VLOG(OVERLAY_DEBUG) << this << ": updating peer " << id << " up to version " << node.version(); + V->update(std::move(node)); + } else { + VLOG(OVERLAY_DEBUG) << this << ": adding peer " << id << " of version " << node.version(); + CHECK(overlay_type_ != OverlayType::CertificatedMembers || (node.certificate() && !node.certificate()->empty())); + peer_list_.peers_.insert(id, OverlayPeer(std::move(node))); + del_some_peers(); + auto X = peer_list_.peers_.get(id); + CHECK(X); + + if (peer_list_.neighbours_.size() < max_neighbours() && + !(X->get_node()->flags() & OverlayMemberFlags::DoNotReceiveBroadcasts) && X->get_id() != local_id_) { + peer_list_.neighbours_.push_back(X->get_id()); + X->set_neighbour(true); + } + + update_neighbours(0); + } } void OverlayImpl::add_peers(std::vector peers) { for (auto &node : peers) { - add_peer_in(std::move(node)); + add_peer(std::move(node)); } } -void OverlayImpl::add_peer(OverlayNode P) { - add_peer_in(std::move(P)); +void OverlayImpl::add_peers(const tl_object_ptr &nodes) { + for (auto &n : nodes->nodes_) { + auto N = OverlayNode::create(n); + if (N.is_ok()) { + add_peer(N.move_as_ok()); + } + } +} + +void OverlayImpl::add_peers(const tl_object_ptr &nodes) { + for (auto &n : nodes->nodes_) { + auto N = OverlayNode::create(n); + if (N.is_ok()) { + add_peer(N.move_as_ok()); + } + } } void OverlayImpl::on_ping_result(adnl::AdnlNodeIdShort peer, bool success) { - if (!public_) { + if (overlay_type_ == OverlayType::FixedMemberList) { return; } - if (OverlayPeer *p = peers_.get(peer)) { + if (OverlayPeer *p = peer_list_.peers_.get(peer)) { p->on_ping_result(success); if (p->is_alive()) { - bad_peers_.erase(peer); + peer_list_.bad_peers_.erase(peer); } else { - bad_peers_.insert(peer); + peer_list_.bad_peers_.insert(peer); } } } void OverlayImpl::receive_random_peers(adnl::AdnlNodeIdShort src, td::Result R) { - CHECK(public_); + CHECK(overlay_type_ != OverlayType::FixedMemberList); on_ping_result(src, R.is_ok()); if (R.is_error()) { VLOG(OVERLAY_NOTICE) << this << ": failed getRandomPeers query: " << R.move_as_error(); @@ -156,16 +277,24 @@ void OverlayImpl::receive_random_peers(adnl::AdnlNodeIdShort src, td::Result nodes; - for (auto &n : res->nodes_) { - auto N = OverlayNode::create(n); - if (N.is_ok()) { - nodes.emplace_back(N.move_as_ok()); - } +void OverlayImpl::receive_random_peers_v2(adnl::AdnlNodeIdShort src, td::Result R) { + CHECK(overlay_type_ != OverlayType::FixedMemberList); + on_ping_result(src, R.is_ok()); + if (R.is_error()) { + VLOG(OVERLAY_NOTICE) << this << ": failed getRandomPeersV2 query: " << R.move_as_error(); + return; } - add_peers(std::move(nodes)); + auto R2 = fetch_tl_object(R.move_as_ok(), true); + if (R2.is_error()) { + VLOG(OVERLAY_WARNING) << this << ": dropping incorrect answer to overlay.getRandomPeers query from " << src << ": " + << R2.move_as_error(); + return; + } + + add_peers(R2.move_as_ok()); } void OverlayImpl::send_random_peers_cont(adnl::AdnlNodeIdShort src, OverlayNode node, @@ -175,10 +304,13 @@ void OverlayImpl::send_random_peers_cont(adnl::AdnlNodeIdShort src, OverlayNode vec.emplace_back(node.tl()); } - for (td::uint32 i = 0; i < nodes_to_send(); i++) { + td::uint32 max_iterations = nodes_to_send() + 16; + for (td::uint32 i = 0; i < max_iterations && vec.size() < nodes_to_send(); i++) { auto P = get_random_peer(true); if (P) { - vec.emplace_back(P->get().tl()); + if (P->has_full_id()) { + vec.emplace_back(P->get_node()->tl()); + } } else { break; } @@ -213,58 +345,110 @@ void OverlayImpl::send_random_peers(adnl::AdnlNodeIdShort src, td::Promise promise) { + std::vector> vec; + if (announce_self_) { + CHECK(is_persistent_node(node.adnl_id_short()) || !node.certificate()->empty()); + vec.emplace_back(node.tl_v2()); + } + + td::uint32 max_iterations = nodes_to_send() + 16; + for (td::uint32 i = 0; i < max_iterations && vec.size() < nodes_to_send(); i++) { + auto P = get_random_peer(true); + if (P) { + if (P->has_full_id() && !P->is_permanent_member()) { + vec.emplace_back(P->get_node()->tl_v2()); + } + } else { + break; + } + } + + if (promise) { + auto Q = create_tl_object(std::move(vec)); + promise.set_value(serialize_tl_object(Q, true)); + } else { + auto P = + td::PromiseCreator::lambda([SelfId = actor_id(this), src, oid = print_id()](td::Result res) { + td::actor::send_closure(SelfId, &OverlayImpl::receive_random_peers_v2, src, std::move(res)); + }); + auto Q = + create_tl_object(create_tl_object(std::move(vec))); + td::actor::send_closure(manager_, &OverlayManager::send_query, src, local_id_, overlay_id_, + "overlay getRandomPeers", std::move(P), + td::Timestamp::in(5.0 + td::Random::fast(0, 50) * 0.1), serialize_tl_object(Q, true)); + } +} + +void OverlayImpl::send_random_peers_v2(adnl::AdnlNodeIdShort src, td::Promise promise) { + auto P = td::PromiseCreator::lambda([src, promise = std::move(promise), + SelfId = actor_id(this)](td::Result res) mutable { + if (res.is_error()) { + promise.set_error(td::Status::Error(ErrorCode::error, "cannot get self node")); + return; + } + td::actor::send_closure(SelfId, &OverlayImpl::send_random_peers_v2_cont, src, res.move_as_ok(), std::move(promise)); + }); + + get_self_node(std::move(P)); +} + void OverlayImpl::update_neighbours(td::uint32 nodes_to_change) { - if (peers_.size() == 0) { + if (peer_list_.peers_.size() == 0) { return; } td::uint32 iter = 0; - while (iter < 10 && (nodes_to_change > 0 || neighbours_.size() < max_neighbours())) { - auto X = peers_.get_random(); + while (iter++ < 10 && (nodes_to_change > 0 || peer_list_.neighbours_.size() < max_neighbours())) { + auto X = peer_list_.peers_.get_random(); if (!X) { break; } if (X->get_id() == local_id_) { - iter++; continue; } - if (public_ && X->get_version() <= td::Clocks::system() - 600) { + if (X->get_version() <= td::Clocks::system() - Overlays::overlay_peer_ttl()) { + if (X->is_permanent_member()) { + del_from_neighbour_list(X); + } else { + auto id = X->get_id(); + del_peer(id); + } + continue; + } + + if (overlay_type_ == OverlayType::CertificatedMembers && !X->is_permanent_member() && + X->certificate()->is_expired()) { + auto id = X->get_id(); + del_peer(id); + continue; + } + + if (X->get_node()->flags() & OverlayMemberFlags::DoNotReceiveBroadcasts) { if (X->is_neighbour()) { - bool found = false; - for (auto &n : neighbours_) { - if (n == X->get_id()) { - n = *neighbours_.rbegin(); - found = true; - break; - } - } - CHECK(found); - neighbours_.pop_back(); - X->set_neighbour(false); + del_from_neighbour_list(X); } - bad_peers_.erase(X->get_id()); - peers_.remove(X->get_id()); continue; } if (X->is_neighbour()) { - iter++; continue; } - if (neighbours_.size() < max_neighbours()) { + if (peer_list_.neighbours_.size() < max_neighbours()) { VLOG(OVERLAY_INFO) << this << ": adding new neighbour " << X->get_id(); - neighbours_.push_back(X->get_id()); + peer_list_.neighbours_.push_back(X->get_id()); X->set_neighbour(true); } else { CHECK(nodes_to_change > 0); - auto i = td::Random::fast(0, static_cast(neighbours_.size()) - 1); - auto Y = peers_.get(neighbours_[i]); + auto i = td::Random::fast(0, static_cast(peer_list_.neighbours_.size()) - 1); + auto Y = peer_list_.peers_.get(peer_list_.neighbours_[i]); CHECK(Y != nullptr); CHECK(Y->is_neighbour()); Y->set_neighbour(false); - neighbours_[i] = X->get_id(); + peer_list_.neighbours_[i] = X->get_id(); X->set_neighbour(true); nodes_to_change--; VLOG(OVERLAY_INFO) << this << ": changing neighbour " << Y->get_id() << " -> " << X->get_id(); @@ -274,9 +458,11 @@ void OverlayImpl::update_neighbours(td::uint32 nodes_to_change) { OverlayPeer *OverlayImpl::get_random_peer(bool only_alive) { size_t skip_bad = 3; - while (peers_.size() > (only_alive ? bad_peers_.size() : 0)) { - auto P = peers_.get_random(); - if (public_ && P->get_version() + 3600 < td::Clocks::system()) { + OverlayPeer *res = nullptr; + while (!res && peer_list_.peers_.size() > (only_alive ? peer_list_.bad_peers_.size() : 0)) { + auto P = peer_list_.peers_.get_random(); + if (!P->is_permanent_member() && + (P->get_version() + 3600 < td::Clocks::system() || P->certificate()->is_expired())) { VLOG(OVERLAY_INFO) << this << ": deleting outdated peer " << P->get_id(); del_peer(P->get_id()); continue; @@ -290,18 +476,19 @@ OverlayPeer *OverlayImpl::get_random_peer(bool only_alive) { continue; } } - return P; + res = P; } - return nullptr; + update_neighbours(0); + return res; } void OverlayImpl::get_overlay_random_peers(td::uint32 max_peers, td::Promise> promise) { std::vector v; auto t = td::Clocks::system(); - while (v.size() < max_peers && v.size() < peers_.size() - bad_peers_.size()) { - auto P = peers_.get_random(); - if (public_ && P->get_version() + 3600 < t) { + while (v.size() < max_peers && v.size() < peer_list_.peers_.size() - peer_list_.bad_peers_.size()) { + auto P = peer_list_.peers_.get_random(); + if (!P->is_permanent_member() && (P->get_version() + 3600 < t || P->certificate()->is_expired(t))) { VLOG(OVERLAY_INFO) << this << ": deleting outdated peer " << P->get_id(); del_peer(P->get_id()); } else if (P->is_alive()) { @@ -317,20 +504,225 @@ void OverlayImpl::get_overlay_random_peers(td::uint32 max_peers, } } } + update_neighbours(0); promise.set_result(std::move(v)); } void OverlayImpl::receive_nodes_from_db(tl_object_ptr tl_nodes) { - if (public_) { - std::vector nodes; - for (auto &n : tl_nodes->nodes_) { - auto N = OverlayNode::create(n); - if (N.is_ok()) { - nodes.emplace_back(N.move_as_ok()); + if (overlay_type_ != OverlayType::FixedMemberList) { + add_peers(tl_nodes); + } +} + +void OverlayImpl::receive_nodes_from_db_v2(tl_object_ptr tl_nodes) { + if (overlay_type_ != OverlayType::FixedMemberList) { + add_peers(tl_nodes); + } +} + +bool OverlayImpl::is_persistent_node(const adnl::AdnlNodeIdShort &id) { + auto P = peer_list_.peers_.get(id); + if (!P) { + return false; + } + return P->is_permanent_member(); +} + +bool OverlayImpl::is_valid_peer(const adnl::AdnlNodeIdShort &src, + const ton_api::overlay_MemberCertificate *certificate) { + if (overlay_type_ == OverlayType::Public) { + on_ping_result(src, true); + return true; + } else if (overlay_type_ == OverlayType::FixedMemberList) { + return peer_list_.peers_.get(src); + } else { + OverlayMemberCertificate cert(certificate); + if (cert.empty()) { + auto P = peer_list_.peers_.get(src); + if (P && !P->is_permanent_member()) { + auto C = P->certificate(); + if (C) { + cert = *C; + } + } + } + + auto S = validate_peer_certificate(src, cert); + if (S.is_error()) { + VLOG(OVERLAY_WARNING) << "adnl=" << src << ": certificate is invalid: " << S; + return false; + } + auto P = peer_list_.peers_.get(src); + if (P) { + CHECK(P->is_permanent_member() || !cert.empty()); + P->update_certificate(std::move(cert)); + } + return true; + } +} + +void OverlayImpl::iterate_all_peers(std::function cb) { + peer_list_.peers_.iterate([&](const adnl::AdnlNodeIdShort &key, OverlayPeer &peer) { cb(key, peer); }); +} + +void OverlayImpl::update_peer_err_ctr(adnl::AdnlNodeIdShort peer_id, bool is_fec) { + auto src_peer = peer_list_.peers_.get(peer_id); + if (src_peer) { + if (is_fec) { + src_peer->fec_broadcast_errors++; + } else { + src_peer->broadcast_errors++; + } + } +} + +void OverlayImpl::update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) { + auto out_peer = peer_list_.peers_.get(peer_id); + if (out_peer) { + out_peer->throughput_out_bytes_ctr += msg_size; + out_peer->throughput_out_packets_ctr++; + + if (is_query) { + out_peer->last_out_query_at = td::Timestamp::now(); + } + } +} + +void OverlayImpl::update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) { + auto in_peer = peer_list_.peers_.get(peer_id); + if (in_peer) { + in_peer->throughput_in_bytes_ctr += msg_size; + in_peer->throughput_in_packets_ctr++; + + if (is_query) { + in_peer->last_in_query_at = td::Timestamp::now(); + } + } +} + +void OverlayImpl::update_peer_ip_str(adnl::AdnlNodeIdShort peer_id, td::string ip_str) { + auto fpeer = peer_list_.peers_.get(peer_id); + if (fpeer) { + fpeer->ip_addr_str = ip_str; + } +} + +bool OverlayImpl::has_good_peers() const { + return peer_list_.peers_.size() > peer_list_.bad_peers_.size(); +} + +bool OverlayImpl::is_root_public_key(const PublicKeyHash &key) const { + return peer_list_.root_public_keys_.count(key) > 0; +} + +std::vector OverlayImpl::get_neighbours(td::uint32 max_size) const { + if (max_size == 0 || max_size >= peer_list_.neighbours_.size()) { + return peer_list_.neighbours_; + } else { + std::vector vec; + std::vector ul; + for (td::uint32 i = 0; i < max_size; i++) { + td::uint32 t = td::Random::fast(0, static_cast(peer_list_.neighbours_.size()) - 1 - i); + td::uint32 j; + for (j = 0; j < i && ul[j] <= t; j++) { + t++; } + ul.emplace(ul.begin() + j, t); + vec.push_back(peer_list_.neighbours_[t]); + } + return vec; + } +} + +void OverlayImpl::send_message_to_neighbours(td::BufferSlice data) { + wait_neighbours_not_empty([this, data = std::move(data)](td::Result R) { + if (R.is_error()) { + return; } - add_peers(std::move(nodes)); + for (auto &n : peer_list_.neighbours_) { + td::actor::send_closure(manager_, &OverlayManager::send_message, n, local_id_, overlay_id_, data.clone()); + } + }); +} + +size_t OverlayImpl::neighbours_cnt() const { + return peer_list_.neighbours_.size(); +} + +void OverlayImpl::update_root_member_list(std::vector ids, + std::vector root_public_keys, OverlayMemberCertificate cert) { + td::uint32 expectd_size = + (td::uint32)(ids.size() + root_public_keys.size() * opts_.max_slaves_in_semiprivate_overlay_); + if (expectd_size > opts_.max_peers_) { + opts_.max_peers_ = expectd_size; } + if (expectd_size > opts_.max_neighbours_) { + opts_.max_neighbours_ = expectd_size; + } + std::sort(ids.begin(), ids.end()); + auto old_root_public_keys = std::move(peer_list_.root_public_keys_); + for (const auto &pub_key : root_public_keys) { + auto it = old_root_public_keys.find(pub_key); + if (it != old_root_public_keys.end()) { + peer_list_.root_public_keys_.emplace(it->first, std::move(it->second)); + } else { + peer_list_.root_public_keys_.emplace(pub_key, PeerList::SlaveKeys{}); + } + } + std::vector to_del; + peer_list_.peers_.iterate([&](const adnl::AdnlNodeIdShort &key, OverlayPeer &peer) { + peer.set_permanent(std::binary_search(ids.begin(), ids.end(), key)); + if (peer.is_permanent_member()) { + peer.clear_certificate(); + } else { + auto S = validate_peer_certificate(peer.get_id(), peer.certificate()); + if (S.is_error()) { + to_del.push_back(peer.get_id()); + } + } + }); + for (const auto &id : to_del) { + del_peer(id); + } + for (const auto &id : ids) { + if (!peer_list_.peers_.exists(id)) { + OverlayNode node(id, overlay_id_, opts_.default_permanent_members_flags_); + OverlayPeer peer(std::move(node)); + peer.set_permanent(true); + CHECK(peer.is_permanent_member()); + peer_list_.peers_.insert(std::move(id), std::move(peer)); + } + } + + update_member_certificate(std::move(cert)); + update_neighbours(0); +} + +void OverlayImpl::update_member_certificate(OverlayMemberCertificate cert) { + peer_list_.cert_ = std::move(cert); + + if (is_persistent_node(local_id_)) { + peer_list_.local_cert_is_valid_until_ = td::Timestamp::in(86400.0 * 365 * 100); /* 100 years */ + } else { + auto R = validate_peer_certificate(local_id_, &peer_list_.cert_); + if (R.is_ok()) { + peer_list_.local_cert_is_valid_until_ = td::Timestamp::at_unix(cert.expire_at()); + } else { + peer_list_.local_cert_is_valid_until_ = td::Timestamp::never(); + } + } +} + +bool OverlayImpl::has_valid_membership_certificate() { + if (overlay_type_ != OverlayType::CertificatedMembers) { + return true; + } + + if (!peer_list_.local_cert_is_valid_until_) { + return false; + } + + return !peer_list_.local_cert_is_valid_until_.is_in_past(); } } // namespace overlay diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index 43b3b7e5a..345d18e68 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -18,6 +18,7 @@ */ #include "auto/tl/ton_api.h" #include "td/utils/Random.h" +#include "common/delay.h" #include "adnl/utils.hpp" #include "dht/dht.h" @@ -26,41 +27,61 @@ #include "auto/tl/ton_api.hpp" #include "keys/encryptor.h" +#include "td/utils/Status.h" #include "td/utils/StringBuilder.h" +#include "td/utils/port/signals.h" +#include namespace ton { namespace overlay { -td::actor::ActorOwn Overlay::create(td::actor::ActorId keyring, - td::actor::ActorId adnl, - td::actor::ActorId manager, - td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, - OverlayIdFull overlay_id, std::unique_ptr callback, - OverlayPrivacyRules rules, td::string scope, OverlayOptions opts) { - auto R = td::actor::create_actor("overlay", keyring, adnl, manager, dht_node, local_id, - std::move(overlay_id), true, std::vector(), - std::move(callback), std::move(rules), scope, opts); +const OverlayMemberCertificate OverlayNode::empty_certificate_{}; + +td::actor::ActorOwn Overlay::create_public(td::actor::ActorId keyring, + td::actor::ActorId adnl, + td::actor::ActorId manager, + td::actor::ActorId dht_node, + adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + std::unique_ptr callback, + OverlayPrivacyRules rules, td::string scope, OverlayOptions opts) { + auto R = td::actor::create_actor( + "overlay", keyring, adnl, manager, dht_node, local_id, std::move(overlay_id), OverlayType::Public, + std::vector(), std::vector(), OverlayMemberCertificate{}, std::move(callback), + std::move(rules), std::move(scope), std::move(opts)); return td::actor::ActorOwn(std::move(R)); } -td::actor::ActorOwn Overlay::create(td::actor::ActorId keyring, - td::actor::ActorId adnl, - td::actor::ActorId manager, - td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, - OverlayIdFull overlay_id, std::vector nodes, - std::unique_ptr callback, OverlayPrivacyRules rules, - std::string scope) { +td::actor::ActorOwn Overlay::create_private( + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId manager, td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, + OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, + OverlayPrivacyRules rules, std::string scope, OverlayOptions opts) { auto R = td::actor::create_actor("overlay", keyring, adnl, manager, dht_node, local_id, - std::move(overlay_id), false, std::move(nodes), std::move(callback), - std::move(rules), std::move(scope)); + std::move(overlay_id), OverlayType::FixedMemberList, std::move(nodes), + std::vector(), OverlayMemberCertificate{}, + std::move(callback), std::move(rules), std::move(scope)); + return td::actor::ActorOwn(std::move(R)); +} + +td::actor::ActorOwn Overlay::create_semiprivate( + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId manager, td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, + OverlayIdFull overlay_id, std::vector nodes, std::vector root_public_keys, + OverlayMemberCertificate cert, std::unique_ptr callback, OverlayPrivacyRules rules, + std::string scope, OverlayOptions opts) { + auto R = td::actor::create_actor( + "overlay", keyring, adnl, manager, dht_node, local_id, std::move(overlay_id), OverlayType::CertificatedMembers, + std::move(nodes), std::move(root_public_keys), std::move(cert), std::move(callback), std::move(rules), + std::move(scope), std::move(opts)); return td::actor::ActorOwn(std::move(R)); } OverlayImpl::OverlayImpl(td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId manager, td::actor::ActorId dht_node, - adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, bool pub, - std::vector nodes, std::unique_ptr callback, + adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, OverlayType overlay_type, + std::vector nodes, std::vector root_public_keys, + OverlayMemberCertificate cert, std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope, OverlayOptions opts) : keyring_(keyring) , adnl_(adnl) @@ -69,37 +90,28 @@ OverlayImpl::OverlayImpl(td::actor::ActorId keyring, td::actor , local_id_(local_id) , id_full_(std::move(overlay_id)) , callback_(std::move(callback)) - , public_(pub) + , overlay_type_(overlay_type) , rules_(std::move(rules)) , scope_(scope) , announce_self_(opts.announce_self_) - , frequent_dht_lookup_(opts.frequent_dht_lookup_) { + , opts_(std::move(opts)) { overlay_id_ = id_full_.compute_short_id(); + frequent_dht_lookup_ = opts_.frequent_dht_lookup_; + peer_list_.local_member_flags_ = opts_.local_overlay_member_flags_; - VLOG(OVERLAY_INFO) << this << ": creating " << (public_ ? "public" : "private"); + VLOG(OVERLAY_INFO) << this << ": creating"; - for (auto &node : nodes) { - CHECK(!public_); - auto X = OverlayNode{node, overlay_id_}; - do_add_peer(std::move(X)); - } + update_root_member_list(std::move(nodes), std::move(root_public_keys), std::move(cert)); update_neighbours(static_cast(nodes.size())); } void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getRandomPeers &query, td::Promise promise) { - if (public_) { + if (overlay_type_ != OverlayType::FixedMemberList) { VLOG(OVERLAY_DEBUG) << this << ": received " << query.peers_->nodes_.size() << " nodes from " << src << " in getRandomPeers query"; - std::vector nodes; - for (auto &n : query.peers_->nodes_) { - auto N = OverlayNode::create(n); - if (N.is_ok()) { - nodes.emplace_back(N.move_as_ok()); - } - } - add_peers(std::move(nodes)); + add_peers(query.peers_); send_random_peers(src, std::move(promise)); } else { VLOG(OVERLAY_WARNING) << this << ": DROPPING getRandomPeers query from " << src << " in private overlay"; @@ -107,6 +119,19 @@ void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getR } } +void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getRandomPeersV2 &query, + td::Promise promise) { + if (overlay_type_ != OverlayType::FixedMemberList) { + VLOG(OVERLAY_DEBUG) << this << ": received " << query.peers_->nodes_.size() << " nodes from " << src + << " in getRandomPeers query"; + add_peers(query.peers_); + send_random_peers_v2(src, std::move(promise)); + } else { + VLOG(OVERLAY_WARNING) << this << ": DROPPING getRandomPeers query from " << src << " in private overlay"; + promise.set_error(td::Status::Error(ErrorCode::protoviolation, "overlay is private")); + } +} + void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcast &query, td::Promise promise) { auto it = broadcasts_.find(query.hash_); @@ -139,17 +164,14 @@ void OverlayImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getB } */ -void OverlayImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise) { - if (!public_) { - auto P = peers_.get(src); - if (P == nullptr) { - VLOG(OVERLAY_WARNING) << this << ": received query in private overlay from unknown source " << src; - promise.set_error(td::Status::Error(ErrorCode::protoviolation, "overlay is private")); - return; - } - } else { - on_ping_result(src, true); +void OverlayImpl::receive_query(adnl::AdnlNodeIdShort src, tl_object_ptr extra, + td::BufferSlice data, td::Promise promise) { + if (!is_valid_peer(src, extra ? extra->certificate_.get() : nullptr)) { + VLOG(OVERLAY_WARNING) << this << ": received query in private overlay from unknown source " << src; + promise.set_error(td::Status::Error(ErrorCode::protoviolation, "overlay is not public")); + return; } + auto R = fetch_tl_object(data.clone(), true); if (R.is_error()) { @@ -167,16 +189,25 @@ void OverlayImpl::receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr bcast) { + if (peer_list_.local_member_flags_ & OverlayMemberFlags::DoNotReceiveBroadcasts) { + return td::Status::OK(); + } return BroadcastSimple::create(this, message_from, std::move(bcast)); } td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr b) { + if (peer_list_.local_member_flags_ & OverlayMemberFlags::DoNotReceiveBroadcasts) { + return td::Status::OK(); + } return OverlayFecBroadcastPart::create(this, message_from, std::move(b)); } td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr b) { + if (peer_list_.local_member_flags_ & OverlayMemberFlags::DoNotReceiveBroadcasts) { + return td::Status::OK(); + } return OverlayFecBroadcastPart::create(this, message_from, std::move(b)); } @@ -188,6 +219,7 @@ td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr msg) { + return td::Status::OK(); // disable this logic for now auto it = fec_broadcasts_.find(msg->hash_); if (it != fec_broadcasts_.end()) { VLOG(OVERLAY_DEBUG) << this << ": received fec opt-out message from " << message_from << " for broadcast " @@ -202,6 +234,7 @@ td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr msg) { + return td::Status::OK(); // disable this logic for now auto it = fec_broadcasts_.find(msg->hash_); if (it != fec_broadcasts_.end()) { VLOG(OVERLAY_DEBUG) << this << ": received fec completed message from " << message_from << " for broadcast " @@ -221,15 +254,13 @@ td::Status OverlayImpl::process_broadcast(adnl::AdnlNodeIdShort message_from, return td::Status::OK(); } -void OverlayImpl::receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice data) { - if (!public_) { - if (peers_.get(src) == nullptr) { - VLOG(OVERLAY_WARNING) << this << ": received query in private overlay from unknown source " << src; - return; - } - } else { - on_ping_result(src, true); +void OverlayImpl::receive_message(adnl::AdnlNodeIdShort src, tl_object_ptr extra, + td::BufferSlice data) { + if (!is_valid_peer(src, extra ? extra->certificate_.get() : nullptr)) { + VLOG(OVERLAY_WARNING) << this << ": received message in private overlay from unknown source " << src; + return; } + auto X = fetch_tl_object(data.clone(), true); if (X.is_error()) { VLOG(OVERLAY_DEBUG) << this << ": received custom message"; @@ -244,44 +275,51 @@ void OverlayImpl::receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice dat void OverlayImpl::alarm() { bcast_gc(); - - if(update_throughput_at_.is_in_past()) { + + if (update_throughput_at_.is_in_past()) { double t_elapsed = td::Time::now() - last_throughput_update_.at(); auto SelfId = actor_id(this); - peers_.iterate([&](const adnl::AdnlNodeIdShort &key, OverlayPeer &peer) { + iterate_all_peers([&](const adnl::AdnlNodeIdShort &key, OverlayPeer &peer) { peer.throughput_out_bytes = static_cast(peer.throughput_out_bytes_ctr / t_elapsed); peer.throughput_in_bytes = static_cast(peer.throughput_in_bytes_ctr / t_elapsed); - + peer.throughput_out_packets = static_cast(peer.throughput_out_packets_ctr / t_elapsed); peer.throughput_in_packets = static_cast(peer.throughput_in_packets_ctr / t_elapsed); - + peer.throughput_out_bytes_ctr = 0; peer.throughput_in_bytes_ctr = 0; - + peer.throughput_out_packets_ctr = 0; peer.throughput_in_packets_ctr = 0; - + auto P = td::PromiseCreator::lambda([SelfId, peer_id = key](td::Result result) { result.ensure(); td::actor::send_closure(SelfId, &Overlay::update_peer_ip_str, peer_id, result.move_as_ok()); }); - + td::actor::send_closure(adnl_, &adnl::AdnlSenderInterface::get_conn_ip_str, local_id_, key, std::move(P)); }); - + update_throughput_at_ = td::Timestamp::in(50.0); last_throughput_update_ = td::Timestamp::now(); } - - if (public_) { - if (peers_.size() > 0) { + + if (overlay_type_ != OverlayType::FixedMemberList) { + if (has_valid_membership_certificate()) { auto P = get_random_peer(); if (P) { - send_random_peers(P->get_id(), {}); + if (overlay_type_ == OverlayType::Public) { + send_random_peers(P->get_id(), {}); + } else { + send_random_peers_v2(P->get_id(), {}); + } } + } else { + VLOG(OVERLAY_WARNING) << "meber certificate ist invalid, valid_until=" + << peer_list_.local_cert_is_valid_until_.at_unix(); } - if (next_dht_query_ && next_dht_query_.is_in_past()) { + if (next_dht_query_ && next_dht_query_.is_in_past() && overlay_type_ == OverlayType::Public) { next_dht_query_ = td::Timestamp::never(); std::function callback = [SelfId = actor_id(this)](dht::DhtValue value) { td::actor::send_closure(SelfId, &OverlayImpl::receive_dht_nodes, std::move(value)); @@ -292,21 +330,22 @@ void OverlayImpl::alarm() { td::actor::send_closure(dht_node_, &dht::Dht::get_value_many, dht::DhtKey{overlay_id_.pubkey_hash(), "nodes", 0}, std::move(callback), std::move(on_finish)); } - if (update_db_at_.is_in_past()) { - if (peers_.size() > 0) { - std::vector vec; - for (td::uint32 i = 0; i < 20; i++) { - auto P = get_random_peer(); - if (!P) { - break; - } - vec.push_back(P->get()); + if (update_db_at_.is_in_past() && overlay_type_ == OverlayType::Public) { + std::vector vec; + for (td::uint32 i = 0; i < 20; i++) { + auto P = get_random_peer(); + if (!P) { + break; } + vec.push_back(P->get_node()->clone()); + } + if (vec.size() > 0) { td::actor::send_closure(manager_, &OverlayManager::save_to_db, local_id_, overlay_id_, std::move(vec)); } update_db_at_ = td::Timestamp::in(60.0); } + update_neighbours(0); alarm_timestamp() = td::Timestamp::in(1.0); } else { update_neighbours(0); @@ -315,7 +354,7 @@ void OverlayImpl::alarm() { } void OverlayImpl::receive_dht_nodes(dht::DhtValue v) { - CHECK(public_); + CHECK(overlay_type_ == OverlayType::Public); auto R = fetch_tl_object(v.value().clone(), true); if (R.is_ok()) { auto r = R.move_as_ok(); @@ -361,7 +400,7 @@ void OverlayImpl::dht_lookup_finished(td::Status S) { } void OverlayImpl::update_dht_nodes(OverlayNode node) { - if (!public_) { + if (overlay_type_ != OverlayType::Public) { return; } @@ -418,21 +457,57 @@ void OverlayImpl::bcast_gc() { CHECK(delivered_broadcasts_.size() == bcast_lru_.size()); } -void OverlayImpl::send_message_to_neighbours(td::BufferSlice data) { - for (auto &n : neighbours_) { - td::actor::send_closure(manager_, &OverlayManager::send_message, n, local_id_, overlay_id_, data.clone()); +void OverlayImpl::wait_neighbours_not_empty(td::Promise promise, int max_retries) { + if (!peer_list_.neighbours_.empty()) { + promise.set_result(td::Unit()); + } else if (max_retries > 0) { + delay_action( + [SelfId = actor_id(this), promise = std::move(promise), max_retries]() mutable { + td::actor::send_closure(SelfId, &OverlayImpl::wait_neighbours_not_empty, std::move(promise), max_retries - 1); + }, + td::Timestamp::in(0.5)); + } else { + promise.set_error(td::Status::Error(ErrorCode::timeout)); } } void OverlayImpl::send_broadcast(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) { - auto S = BroadcastSimple::create_new(actor_id(this), keyring_, send_as, std::move(data), flags); - if (S.is_error()) { - LOG(WARNING) << "failed to send broadcast: " << S; + if (!has_valid_membership_certificate()) { + VLOG(OVERLAY_WARNING) << "member certificate is invalid, valid_until=" + << peer_list_.local_cert_is_valid_until_.at_unix(); + return; + } + if (!has_valid_broadcast_certificate(send_as, data.size(), false)) { + VLOG(OVERLAY_WARNING) << "broadcast source certificate is invalid"; + return; } + wait_neighbours_not_empty([this, send_as, flags, data = std::move(data)](td::Result R) mutable { + if (R.is_error()) { + return; + } + auto S = BroadcastSimple::create_new(actor_id(this), keyring_, send_as, std::move(data), flags); + if (S.is_error()) { + LOG(WARNING) << "failed to send broadcast: " << S; + } + }); } void OverlayImpl::send_broadcast_fec(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) { - OverlayOutboundFecBroadcast::create(std::move(data), flags, actor_id(this), send_as); + if (!has_valid_membership_certificate()) { + VLOG(OVERLAY_WARNING) << "meber certificate ist invalid, valid_until=" + << peer_list_.local_cert_is_valid_until_.at_unix(); + return; + } + if (!has_valid_broadcast_certificate(send_as, data.size(), true)) { + VLOG(OVERLAY_WARNING) << "broadcast source certificate is invalid"; + return; + } + wait_neighbours_not_empty([this, send_as, flags, data = std::move(data)](td::Result R) mutable { + if (R.is_error()) { + return; + } + OverlayOutboundFecBroadcast::create(std::move(data), flags, actor_id(this), send_as); + }); } void OverlayImpl::print(td::StringBuilder &sb) { @@ -450,6 +525,22 @@ td::Status OverlayImpl::check_date(td::uint32 date) { return td::Status::OK(); } +BroadcastCheckResult OverlayImpl::check_source_eligible(const PublicKeyHash &source, const Certificate *cert, + td::uint32 size, bool is_fec) { + if (size == 0) { + return BroadcastCheckResult::Forbidden; + } + + auto r = rules_.check_rules(source, size, is_fec); + if (!cert || r == BroadcastCheckResult::Allowed) { + return r; + } + + auto r2 = cert->check(source, overlay_id_, static_cast(td::Clocks::system()), size, is_fec); + r2 = broadcast_check_result_min(r2, rules_.check_rules(cert->issuer_hash(), size, is_fec)); + return broadcast_check_result_max(r, r2); +} + BroadcastCheckResult OverlayImpl::check_source_eligible(PublicKey source, const Certificate *cert, td::uint32 size, bool is_fec) { if (size == 0) { @@ -457,7 +548,7 @@ BroadcastCheckResult OverlayImpl::check_source_eligible(PublicKey source, const } auto short_id = source.compute_short_id(); - auto r = rules_.check_rules(source.compute_short_id(), size, is_fec); + auto r = rules_.check_rules(short_id, size, is_fec); if (!cert || r == BroadcastCheckResult::Allowed) { return r; } @@ -492,21 +583,23 @@ void OverlayImpl::register_fec_broadcast(std::unique_ptr bcast) { } void OverlayImpl::get_self_node(td::Promise promise) { - OverlayNode s{local_id_, overlay_id_}; + OverlayNode s{local_id_, overlay_id_, peer_list_.local_member_flags_}; auto to_sign = s.to_sign(); - auto P = td::PromiseCreator::lambda([oid = print_id(), s = std::move(s), promise = std::move(promise)]( - td::Result> R) mutable { - if (R.is_error()) { - auto S = R.move_as_error(); - LOG(ERROR) << oid << ": failed to get self node: " << S; - promise.set_error(std::move(S)); - return; - } - auto V = R.move_as_ok(); - s.update_signature(std::move(V.first)); - s.update_adnl_id(adnl::AdnlNodeIdFull{V.second}); - promise.set_value(std::move(s)); - }); + auto P = td::PromiseCreator::lambda( + [oid = print_id(), s = std::move(s), cert = peer_list_.cert_, + promise = std::move(promise)](td::Result> R) mutable { + if (R.is_error()) { + auto S = R.move_as_error(); + LOG(ERROR) << oid << ": failed to get self node: " << S; + promise.set_error(std::move(S)); + return; + } + auto V = R.move_as_ok(); + s.update_signature(std::move(V.first)); + s.update_adnl_id(adnl::AdnlNodeIdFull{V.second}); + s.update_certificate(std::move(cert)); + promise.set_value(std::move(s)); + }); td::actor::send_closure(keyring_, &keyring::Keyring::sign_add_get_public_key, local_id_.pubkey_hash(), std::move(to_sign), std::move(P)); @@ -598,17 +691,6 @@ void OverlayImpl::check_broadcast(PublicKeyHash src, td::BufferSlice data, td::P callback_->check_broadcast(src, overlay_id_, std::move(data), std::move(promise)); } -void OverlayImpl::update_peer_err_ctr(adnl::AdnlNodeIdShort peer_id, bool is_fec) { - auto src_peer = peers_.get(peer_id); - if(src_peer) { - if(is_fec) { - src_peer->fec_broadcast_errors++; - } else { - src_peer->broadcast_errors++; - } - } -} - void OverlayImpl::broadcast_checked(Overlay::BroadcastHash hash, td::Result R) { { auto it = broadcasts_.find(hash); @@ -630,30 +712,47 @@ void OverlayImpl::get_stats(td::Promiseoverlay_id_ = overlay_id_.bits256_value(); res->overlay_id_full_ = id_full_.pubkey().tl(); res->scope_ = scope_; - peers_.iterate([&](const adnl::AdnlNodeIdShort &key, const OverlayPeer &peer) { + iterate_all_peers([&](const adnl::AdnlNodeIdShort &key, const OverlayPeer &peer) { auto node_obj = create_tl_object(); node_obj->adnl_id_ = key.bits256_value(); node_obj->t_out_bytes_ = peer.throughput_out_bytes; node_obj->t_in_bytes_ = peer.throughput_in_bytes; - + node_obj->t_out_pckts_ = peer.throughput_out_packets; node_obj->t_in_pckts_ = peer.throughput_in_packets; - + node_obj->ip_addr_ = peer.ip_addr_str; - + node_obj->last_in_query_ = static_cast(peer.last_in_query_at.at_unix()); node_obj->last_out_query_ = static_cast(peer.last_out_query_at.at_unix()); - + node_obj->bdcst_errors_ = peer.broadcast_errors; node_obj->fec_bdcst_errors_ = peer.fec_broadcast_errors; - + + node_obj->is_neighbour_ = peer.is_neighbour(); + node_obj->is_alive_ = peer.is_alive(); + node_obj->node_flags_ = peer.get_node()->flags(); + res->nodes_.push_back(std::move(node_obj)); }); res->stats_.push_back( - create_tl_object("neighbours_cnt", PSTRING() << neighbours_.size())); + create_tl_object("neighbours_cnt", PSTRING() << neighbours_cnt())); - promise.set_value(std::move(res)); + callback_->get_stats_extra([promise = std::move(promise), res = std::move(res)](td::Result R) mutable { + if (R.is_ok()) { + res->extra_ = R.move_as_ok(); + } + promise.set_value(std::move(res)); + }); +} + +bool OverlayImpl::has_valid_broadcast_certificate(const PublicKeyHash &source, size_t size, bool is_fec) { + if (size > std::numeric_limits::max()) { + return false; + } + auto it = certs_.find(source); + return check_source_eligible(source, it == certs_.end() ? nullptr : it->second.get(), (td::uint32)size, is_fec); } } // namespace overlay diff --git a/overlay/overlay.h b/overlay/overlay.h index c1fe9643c..f25db206a 100644 --- a/overlay/overlay.h +++ b/overlay/overlay.h @@ -18,6 +18,7 @@ */ #pragma once +#include "auto/tl/ton_api.h" #include "td/utils/buffer.h" #include "td/utils/int_types.h" @@ -37,24 +38,29 @@ class Overlay : public td::actor::Actor { using BroadcastDataHash = td::Bits256; using BroadcastPartHash = td::Bits256; - static td::actor::ActorOwn create(td::actor::ActorId keyring, - td::actor::ActorId adnl, - td::actor::ActorId manager, - td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, - OverlayIdFull overlay_id, std::unique_ptr callback, - OverlayPrivacyRules rules, td::string scope, OverlayOptions opts = {}); - static td::actor::ActorOwn create(td::actor::ActorId keyring, - td::actor::ActorId adnl, - td::actor::ActorId manager, - td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, - OverlayIdFull overlay_id, std::vector nodes, - std::unique_ptr callback, OverlayPrivacyRules rules, - std::string scope); + static td::actor::ActorOwn create_public( + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId manager, td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, + OverlayIdFull overlay_id, std::unique_ptr callback, OverlayPrivacyRules rules, + td::string scope, OverlayOptions opts = {}); + static td::actor::ActorOwn create_private( + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId manager, td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, + OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, + OverlayPrivacyRules rules, std::string scope, OverlayOptions opts = {}); + static td::actor::ActorOwn create_semiprivate( + td::actor::ActorId keyring, td::actor::ActorId adnl, + td::actor::ActorId manager, td::actor::ActorId dht_node, adnl::AdnlNodeIdShort local_id, + OverlayIdFull overlay_id, std::vector nodes, std::vector root_public_keys, + OverlayMemberCertificate cert, std::unique_ptr callback, OverlayPrivacyRules rules, + std::string scope, OverlayOptions opts = {}); virtual void update_dht_node(td::actor::ActorId dht) = 0; - virtual void receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice data) = 0; - virtual void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise) = 0; + virtual void receive_message(adnl::AdnlNodeIdShort src, tl_object_ptr extra, + td::BufferSlice data) = 0; + virtual void receive_query(adnl::AdnlNodeIdShort src, tl_object_ptr extra, + td::BufferSlice data, td::Promise promise) = 0; virtual void send_message_to_neighbours(td::BufferSlice data) = 0; virtual void send_broadcast(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) = 0; virtual void send_broadcast_fec(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) = 0; @@ -64,12 +70,17 @@ class Overlay : public td::actor::Actor { virtual void add_certificate(PublicKeyHash key, std::shared_ptr) = 0; virtual void set_privacy_rules(OverlayPrivacyRules rules) = 0; virtual void receive_nodes_from_db(tl_object_ptr nodes) = 0; + virtual void receive_nodes_from_db_v2(tl_object_ptr nodes) = 0; virtual void get_stats(td::Promise> promise) = 0; virtual void update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) = 0; virtual void update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) = 0; virtual void update_peer_ip_str(adnl::AdnlNodeIdShort peer_id, td::string ip_str) = 0; + virtual void update_member_certificate(OverlayMemberCertificate cert) = 0; + virtual void update_root_member_list(std::vector nodes, + std::vector root_public_keys, OverlayMemberCertificate cert) = 0; //virtual void receive_broadcast(td::BufferSlice data) = 0; //virtual void subscribe(std::unique_ptr callback) = 0; + virtual void forget_peer(adnl::AdnlNodeIdShort peer_id) = 0; }; } // namespace overlay diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index 8fb3d91d7..b08aaf460 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -18,11 +18,15 @@ */ #pragma once +#include +#include #include #include #include +#include #include +#include "adnl/adnl-node-id.hpp" #include "overlay.h" #include "overlay-manager.h" #include "overlay-fec.hpp" @@ -32,6 +36,9 @@ #include "td/utils/DecTree.h" #include "td/utils/List.h" +#include "td/utils/Status.h" +#include "td/utils/Time.h" +#include "td/utils/buffer.h" #include "td/utils/overloaded.h" #include "fec/fec.h" @@ -40,6 +47,8 @@ #include "auto/tl/ton_api.h" #include "auto/tl/ton_api.hpp" +#include "td/utils/port/signals.h" +#include "tl-utils/common-utils.hpp" namespace ton { @@ -58,15 +67,17 @@ class OverlayPeer { adnl::AdnlNodeIdFull get_full_id() const { return node_.adnl_id_full(); } - OverlayNode get() const { - return node_.clone(); + const OverlayNode *get_node() const { + return &node_; } void update(OverlayNode node) { CHECK(get_id() == node.adnl_id_short()); - if (node.version() > node_.version()) { - node_ = std::move(node); - } + node_.update(std::move(node)); + } + void update_certificate(OverlayMemberCertificate cert) { + node_.update_certificate(std::move(cert)); } + OverlayPeer(OverlayNode node) : node_(std::move(node)) { id_ = node_.adnl_id_short(); } @@ -95,24 +106,44 @@ class OverlayPeer { return is_alive_; } + bool is_permanent_member() const { + return is_permanent_member_; + } + + void set_permanent(bool value) { + is_permanent_member_ = value; + } + + void clear_certificate() { + node_.clear_certificate(); + } + + auto certificate() const { + return node_.certificate(); + } + + bool has_full_id() const { + return node_.has_full_id(); + } + td::uint32 throughput_out_bytes = 0; td::uint32 throughput_in_bytes = 0; - + td::uint32 throughput_out_packets = 0; td::uint32 throughput_in_packets = 0; - + td::uint32 throughput_out_bytes_ctr = 0; td::uint32 throughput_in_bytes_ctr = 0; - + td::uint32 throughput_out_packets_ctr = 0; td::uint32 throughput_in_packets_ctr = 0; - + td::uint32 broadcast_errors = 0; td::uint32 fec_broadcast_errors = 0; - + td::Timestamp last_in_query_at = td::Timestamp::now(); td::Timestamp last_out_query_at = td::Timestamp::now(); - + td::string ip_addr_str = "undefined"; private: @@ -122,6 +153,7 @@ class OverlayPeer { bool is_neighbour_ = false; size_t missed_pings_ = 0; bool is_alive_ = true; + bool is_permanent_member_ = false; td::Timestamp last_ping_at_ = td::Timestamp::now(); }; @@ -129,19 +161,23 @@ class OverlayImpl : public Overlay { public: OverlayImpl(td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId manager, td::actor::ActorId dht_node, - adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, bool pub, - std::vector nodes, std::unique_ptr callback, - OverlayPrivacyRules rules, td::string scope = "{ \"type\": \"undefined\" }", OverlayOptions opts = {}); + adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, OverlayType overlay_type, + std::vector nodes, std::vector root_public_keys, + OverlayMemberCertificate cert, std::unique_ptr callback, OverlayPrivacyRules rules, + td::string scope = "{ \"type\": \"undefined\" }", OverlayOptions opts = {}); void update_dht_node(td::actor::ActorId dht) override { dht_node_ = dht; } - void receive_message(adnl::AdnlNodeIdShort src, td::BufferSlice data) override; - void receive_query(adnl::AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise) override; + void receive_message(adnl::AdnlNodeIdShort src, tl_object_ptr extra, + td::BufferSlice data) override; + void receive_query(adnl::AdnlNodeIdShort src, tl_object_ptr extra, + td::BufferSlice data, td::Promise promise) override; void send_message_to_neighbours(td::BufferSlice data) override; void send_broadcast(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) override; void send_broadcast_fec(PublicKeyHash send_as, td::uint32 flags, td::BufferSlice data) override; void receive_nodes_from_db(tl_object_ptr nodes) override; + void receive_nodes_from_db_v2(tl_object_ptr nodes) override; void get_self_node(td::Promise promise); @@ -149,8 +185,8 @@ class OverlayImpl : public Overlay { void start_up() override { update_throughput_at_ = td::Timestamp::in(50.0); last_throughput_update_ = td::Timestamp::now(); - - if (public_) { + + if (overlay_type_ == OverlayType::Public) { update_db_at_ = td::Timestamp::in(60.0); } alarm_timestamp() = td::Timestamp::in(1); @@ -158,13 +194,17 @@ class OverlayImpl : public Overlay { void on_ping_result(adnl::AdnlNodeIdShort peer, bool success); void receive_random_peers(adnl::AdnlNodeIdShort src, td::Result R); + void receive_random_peers_v2(adnl::AdnlNodeIdShort src, td::Result R); void send_random_peers(adnl::AdnlNodeIdShort dst, td::Promise promise); + void send_random_peers_v2(adnl::AdnlNodeIdShort dst, td::Promise promise); void send_random_peers_cont(adnl::AdnlNodeIdShort dst, OverlayNode node, td::Promise promise); + void send_random_peers_v2_cont(adnl::AdnlNodeIdShort dst, OverlayNode node, td::Promise promise); void get_overlay_random_peers(td::uint32 max_peers, td::Promise> promise) override; void set_privacy_rules(OverlayPrivacyRules rules) override; void add_certificate(PublicKeyHash key, std::shared_ptr cert) override { certs_[key] = std::move(cert); } + void update_member_certificate(OverlayMemberCertificate cert) override; void receive_dht_nodes(dht::DhtValue v); void dht_lookup_finished(td::Status S); @@ -188,6 +228,8 @@ class OverlayImpl : public Overlay { td::Status check_date(td::uint32 date); BroadcastCheckResult check_source_eligible(PublicKey source, const Certificate *cert, td::uint32 size, bool is_fec); + BroadcastCheckResult check_source_eligible(const PublicKeyHash &source, const Certificate *cert, td::uint32 size, + bool is_fec); td::Status check_delivered(BroadcastHash hash); void broadcast_checked(Overlay::BroadcastHash hash, td::Result R); @@ -206,17 +248,7 @@ class OverlayImpl : public Overlay { void send_new_fec_broadcast_part(PublicKeyHash local_id, Overlay::BroadcastDataHash data_hash, td::uint32 size, td::uint32 flags, td::BufferSlice part, td::uint32 seqno, fec::FecType fec_type, td::uint32 date); - std::vector get_neighbours(td::uint32 max_size = 0) const { - if (max_size == 0 || max_size >= neighbours_.size()) { - return neighbours_; - } else { - std::vector vec; - for (td::uint32 i = 0; i < max_size; i++) { - vec.push_back(neighbours_[td::Random::fast(0, static_cast(neighbours_.size()) - 1)]); - } - return vec; - } - } + std::vector get_neighbours(td::uint32 max_size = 0) const; td::actor::ActorId overlay_manager() const { return manager_; } @@ -236,40 +268,60 @@ class OverlayImpl : public Overlay { td::Result get_encryptor(PublicKey source); void get_stats(td::Promise> promise) override; - - void update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) override { - auto out_peer = peers_.get(peer_id); - if(out_peer) { - out_peer->throughput_out_bytes_ctr += msg_size; - out_peer->throughput_out_packets_ctr++; - - if(is_query) - { - out_peer->last_out_query_at = td::Timestamp::now(); - } - } + + void update_throughput_out_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) override; + + void update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) override; + + void update_peer_ip_str(adnl::AdnlNodeIdShort peer_id, td::string ip_str) override; + + void update_root_member_list(std::vector nodes, std::vector root_public_keys, + OverlayMemberCertificate cert) override; + + bool is_valid_peer(const adnl::AdnlNodeIdShort &id, const ton_api::overlay_MemberCertificate *certificate); + bool is_persistent_node(const adnl::AdnlNodeIdShort &id); + + td::uint32 max_data_bcasts() const { + return 100; } - - void update_throughput_in_ctr(adnl::AdnlNodeIdShort peer_id, td::uint32 msg_size, bool is_query) override { - auto in_peer = peers_.get(peer_id); - if(in_peer) { - in_peer->throughput_in_bytes_ctr += msg_size; - in_peer->throughput_in_packets_ctr++; - - if(is_query) - { - in_peer->last_in_query_at = td::Timestamp::now(); - } - } + td::uint32 max_bcasts() const { + return 1000; } - - void update_peer_ip_str(adnl::AdnlNodeIdShort peer_id, td::string ip_str) override { - auto fpeer = peers_.get(peer_id); - if(fpeer) { - fpeer->ip_addr_str = ip_str; - } + td::uint32 max_fec_bcasts() const { + return 20; + } + td::uint32 max_sources() const { + return 10; + } + td::uint32 max_encryptors() const { + return 16; + } + + td::uint32 max_neighbours() const { + return opts_.max_neighbours_; + } + + td::uint32 max_peers() const { + return opts_.max_peers_; } + td::uint32 nodes_to_send() const { + return opts_.nodes_to_send_; + } + + td::uint32 propagate_broadcast_to() const { + return opts_.propagate_broadcast_to_; + } + + bool has_valid_membership_certificate(); + bool has_valid_broadcast_certificate(const PublicKeyHash &source, size_t size, bool is_fec); + + void forget_peer(adnl::AdnlNodeIdShort peer_id) override { + del_peer(peer_id); + } + + void wait_neighbours_not_empty(td::Promise promise, int max_retries = 20); + private: template void process_query(adnl::AdnlNodeIdShort src, T &query, td::Promise promise) { @@ -278,6 +330,8 @@ class OverlayImpl : public Overlay { void process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getRandomPeers &query, td::Promise promise); + void process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getRandomPeersV2 &query, + td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcast &query, td::Promise promise); void process_query(adnl::AdnlNodeIdShort src, ton_api::overlay_getBroadcastList &query, @@ -294,20 +348,28 @@ class OverlayImpl : public Overlay { td::Status process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr msg); td::Status process_broadcast(adnl::AdnlNodeIdShort message_from, tl_object_ptr msg); - void do_add_peer(OverlayNode node); - void add_peer_in_cont(OverlayNode node); - void add_peer_in(OverlayNode node); + td::Status validate_peer_certificate(const adnl::AdnlNodeIdShort &node, const OverlayMemberCertificate &cert); + td::Status validate_peer_certificate(const adnl::AdnlNodeIdShort &node, const OverlayMemberCertificate *cert); + td::Status validate_peer_certificate(const adnl::AdnlNodeIdShort &node, ton_api::overlay_MemberCertificate *cert); void add_peer(OverlayNode node); void add_peers(std::vector nodes); + void add_peers(const tl_object_ptr &nodes); + void add_peers(const tl_object_ptr &nodes); void del_some_peers(); - void del_peer(adnl::AdnlNodeIdShort id); + void del_peer(const adnl::AdnlNodeIdShort &id); + void del_from_neighbour_list(OverlayPeer *P); + void del_from_neighbour_list(const adnl::AdnlNodeIdShort &id); + void iterate_all_peers(std::function cb); OverlayPeer *get_random_peer(bool only_alive = false); + bool is_root_public_key(const PublicKeyHash &key) const; + bool has_good_peers() const; + size_t neighbours_cnt() const; void finish_dht_query() { if (!next_dht_store_query_) { next_dht_store_query_ = td::Timestamp::in(td::Random::fast(60.0, 100.0)); } - if (frequent_dht_lookup_ && peers_.size() == bad_peers_.size()) { + if (frequent_dht_lookup_ && !has_good_peers()) { next_dht_query_ = td::Timestamp::in(td::Random::fast(6.0, 10.0)); } else { next_dht_query_ = next_dht_store_query_; @@ -322,14 +384,11 @@ class OverlayImpl : public Overlay { OverlayIdFull id_full_; OverlayIdShort overlay_id_; - td::DecTree peers_; td::Timestamp next_dht_query_ = td::Timestamp::in(1.0); td::Timestamp next_dht_store_query_ = td::Timestamp::in(1.0); td::Timestamp update_db_at_; td::Timestamp update_throughput_at_; td::Timestamp last_throughput_update_; - std::set bad_peers_; - adnl::AdnlNodeIdShort next_bad_peer_ = adnl::AdnlNodeIdShort::zero(); std::unique_ptr callback_; @@ -337,7 +396,6 @@ class OverlayImpl : public Overlay { std::map> fec_broadcasts_; std::set delivered_broadcasts_; - std::vector neighbours_; td::ListNode bcast_data_lru_; td::ListNode bcast_fec_lru_; std::queue bcast_lru_; @@ -346,33 +404,6 @@ class OverlayImpl : public Overlay { void bcast_gc(); - static td::uint32 max_data_bcasts() { - return 100; - } - static td::uint32 max_bcasts() { - return 1000; - } - static td::uint32 max_fec_bcasts() { - return 20; - } - static td::uint32 max_sources() { - return 10; - } - static td::uint32 max_neighbours() { - return 5; - } - static td::uint32 max_encryptors() { - return 16; - } - - static td::uint32 max_peers() { - return 20; - } - - static td::uint32 nodes_to_send() { - return 4; - } - static BroadcastHash get_broadcast_hash(adnl::AdnlNodeIdShort &src, td::Bits256 &data_hash) { td::uint8 buf[64]; td::MutableSlice m{buf, 64}; @@ -382,8 +413,7 @@ class OverlayImpl : public Overlay { return td::sha256_bits256(td::Slice(buf, 64)); } - bool public_; - bool semi_public_ = false; + OverlayType overlay_type_; OverlayPrivacyRules rules_; td::string scope_; bool announce_self_ = true; @@ -412,6 +442,25 @@ class OverlayImpl : public Overlay { td::ListNode encryptor_lru_; std::map> encryptor_map_; + + struct PeerList { + struct SlaveKey { + td::int32 expire_at{0}; + adnl::AdnlNodeIdShort node{}; + }; + using SlaveKeys = std::vector; + std::map root_public_keys_; + OverlayMemberCertificate cert_; + std::set bad_peers_; + adnl::AdnlNodeIdShort next_bad_peer_ = adnl::AdnlNodeIdShort::zero(); + td::DecTree peers_; + std::vector neighbours_; + + td::Timestamp local_cert_is_valid_until_; + td::uint32 local_member_flags_{0}; + } peer_list_; + + OverlayOptions opts_; }; } // namespace overlay diff --git a/overlay/overlays.h b/overlay/overlays.h index 6bf5852f2..cc112bc43 100644 --- a/overlay/overlays.h +++ b/overlay/overlays.h @@ -18,7 +18,9 @@ */ #pragma once +#include "adnl/adnl-node-id.hpp" #include "adnl/adnl.h" +#include "auto/tl/ton_api.h" #include "dht/dht.h" #include "td/actor/PromiseFuture.h" @@ -33,6 +35,8 @@ namespace ton { namespace overlay { +enum class OverlayType { Public, FixedMemberList, CertificatedMembers }; + class OverlayIdShort { public: OverlayIdShort() { @@ -88,6 +92,10 @@ struct CertificateFlags { enum Values : td::uint32 { AllowFec = 1, Trusted = 2 }; }; +struct OverlayMemberFlags { + enum Values : td::uint32 { DoNotReceiveBroadcasts = 1 }; +}; + enum BroadcastCheckResult { Forbidden = 1, NeedCheck = 2, Allowed = 3 }; inline BroadcastCheckResult broadcast_check_result_max(BroadcastCheckResult l, BroadcastCheckResult r) { @@ -108,7 +116,6 @@ class OverlayPrivacyRules { } BroadcastCheckResult check_rules(PublicKeyHash hash, td::uint32 size, bool is_fec) { - auto it = authorized_keys_.find(hash); if (it == authorized_keys_.end()) { if (size > max_unath_size_) { @@ -158,9 +165,110 @@ class Certificate { td::SharedSlice signature_; }; +class OverlayMemberCertificate { + public: + OverlayMemberCertificate() { + expire_at_ = std::numeric_limits::max(); + } + OverlayMemberCertificate(PublicKey signed_by, td::uint32 flags, td::int32 slot, td::int32 expire_at, + td::BufferSlice signature) + : signed_by_(std::move(signed_by)) + , flags_(flags) + , slot_(slot) + , expire_at_(expire_at) + , signature_(std::move(signature)) { + } + OverlayMemberCertificate(const OverlayMemberCertificate &other) + : signed_by_(other.signed_by_) + , flags_(other.flags_) + , slot_(other.slot_) + , expire_at_(other.expire_at_) + , signature_(other.signature_.clone()) { + } + OverlayMemberCertificate(OverlayMemberCertificate &&) = default; + OverlayMemberCertificate &operator=(OverlayMemberCertificate &&) = default; + OverlayMemberCertificate &operator=(const OverlayMemberCertificate &other) { + signed_by_ = other.signed_by_; + flags_ = other.flags_; + slot_ = other.slot_; + expire_at_ = other.expire_at_; + signature_ = other.signature_.clone(); + return *this; + } + explicit OverlayMemberCertificate(const ton_api::overlay_MemberCertificate *cert); + td::Status check_signature(const adnl::AdnlNodeIdShort &node); + + bool is_expired() const { + return expire_at_ < td::Clocks::system() - 3; + } + + bool is_expired(double cur_time) const { + return expire_at_ < cur_time - 3; + } + + tl_object_ptr tl() const { + if (empty()) { + return create_tl_object(); + } + return create_tl_object(signed_by_.tl(), flags_, slot_, expire_at_, + signature_.clone_as_buffer_slice()); + } + + const auto &issued_by() const { + return signed_by_; + } + + td::Slice signature() const { + return signature_.as_slice(); + } + + td::BufferSlice to_sign_data(const adnl::AdnlNodeIdShort &node) const { + return ton::create_serialize_tl_object(node.tl(), flags_, slot_, + expire_at_); + } + + bool empty() const { + return signed_by_.empty(); + } + + bool is_newer(const OverlayMemberCertificate &other) const { + return !empty() && expire_at_ > other.expire_at_; + } + + auto slot() const { + return slot_; + } + + auto expire_at() const { + return expire_at_; + } + + void set_signature(td::Slice signature) { + signature_ = td::SharedSlice(signature); + } + void set_signature(td::SharedSlice signature) { + signature_ = std::move(signature); + } + + private: + PublicKey signed_by_; + td::uint32 flags_; + td::int32 slot_; + td::int32 expire_at_ = std::numeric_limits::max(); + td::SharedSlice signature_; +}; + + struct OverlayOptions { bool announce_self_ = true; bool frequent_dht_lookup_ = false; + td::uint32 local_overlay_member_flags_ = 0; + td::int32 max_slaves_in_semiprivate_overlay_ = 5; + td::uint32 max_peers_ = 20; + td::uint32 max_neighbours_ = 5; + td::uint32 nodes_to_send_ = 4; + td::uint32 propagate_broadcast_to_ = 5; + td::uint32 default_permanent_members_flags_ = 0; }; class Overlays : public td::actor::Actor { @@ -175,6 +283,9 @@ class Overlays : public td::actor::Actor { td::Promise promise) { promise.set_value(td::Unit()); } + virtual void get_stats_extra(td::Promise promise) { + promise.set_result(""); + } virtual ~Callback() = default; }; @@ -192,6 +303,10 @@ class Overlays : public td::actor::Actor { return 1; } + static constexpr td::uint32 overlay_peer_ttl() { + return 600; + } + static td::actor::ActorOwn create(std::string db_root, td::actor::ActorId keyring, td::actor::ActorId adnl, td::actor::ActorId dht); @@ -201,11 +316,20 @@ class Overlays : public td::actor::Actor { std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope) = 0; virtual void create_public_overlay_ex(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, - std::unique_ptr callback, OverlayPrivacyRules rules, - td::string scope, OverlayOptions opts) = 0; + std::unique_ptr callback, OverlayPrivacyRules rules, td::string scope, + OverlayOptions opts) = 0; + virtual void create_semiprivate_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + std::vector nodes, + std::vector root_public_keys, + OverlayMemberCertificate certificate, + std::unique_ptr callback, OverlayPrivacyRules rules, + td::string scope, OverlayOptions opts) = 0; virtual void create_private_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, std::vector nodes, std::unique_ptr callback, OverlayPrivacyRules rules, std::string scope) = 0; + virtual void create_private_overlay_ex(adnl::AdnlNodeIdShort local_id, OverlayIdFull overlay_id, + std::vector nodes, std::unique_ptr callback, + OverlayPrivacyRules rules, std::string scope, OverlayOptions opts) = 0; virtual void delete_overlay(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id) = 0; virtual void send_query(adnl::AdnlNodeIdShort dst, adnl::AdnlNodeIdShort src, OverlayIdShort overlay_id, @@ -239,9 +363,18 @@ class Overlays : public td::actor::Actor { virtual void update_certificate(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, PublicKeyHash key, std::shared_ptr cert) = 0; + virtual void update_member_certificate(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, + OverlayMemberCertificate certificate) = 0; + virtual void update_root_member_list(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay_id, + std::vector nodes, + std::vector root_public_keys, + OverlayMemberCertificate certificate) = 0; + virtual void get_overlay_random_peers(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, td::uint32 max_peers, td::Promise> promise) = 0; virtual void get_stats(td::Promise> promise) = 0; + + virtual void forget_peer(adnl::AdnlNodeIdShort local_id, OverlayIdShort overlay, adnl::AdnlNodeIdShort peer_id) = 0; }; } // namespace overlay diff --git a/test/test-overlay.cpp b/test/test-overlay.cpp new file mode 100644 index 000000000..31db3e780 --- /dev/null +++ b/test/test-overlay.cpp @@ -0,0 +1,450 @@ +/* + This file is part of TON Blockchain source code. + + TON Blockchain is free software; you can redistribute it and/or + modify it under the terms of the GNU General Public License + as published by the Free Software Foundation; either version 2 + of the License, or (at your option) any later version. + + TON Blockchain is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with TON Blockchain. If not, see . + + In addition, as a special exception, the copyright holders give permission + to link the code of portions of this program with the OpenSSL library. + You must obey the GNU General Public License in all respects for all + of the code used other than OpenSSL. If you modify file(s) with this + exception, you may extend this exception to your version of the file(s), + but you are not obligated to do so. If you do not wish to do so, delete this + exception statement from your version. If you delete this exception statement + from all source files in the program, then also delete it here. + + Copyright 2017-2020 Telegram Systems LLP +*/ +#include "adnl/adnl-node-id.hpp" +#include "adnl/adnl.h" +#include "adnl/utils.hpp" +#include "adnl/adnl-test-loopback-implementation.h" +#include "auto/tl/ton_api.h" +#include "checksum.h" +#include "common/bitstring.h" +#include "dht/dht.h" +#include "keys/keys.hpp" +#include "overlay-manager.h" +#include "overlay.h" +#include "overlay-id.hpp" +#include "overlay/overlays.h" +#include "td/actor/actor.h" +#include "td/utils/OptionParser.h" +#include "td/utils/Status.h" +#include "td/utils/Time.h" +#include "td/utils/UInt.h" +#include "td/utils/buffer.h" +#include "td/utils/crypto.h" +#include "td/utils/filesystem.h" +#include "td/utils/format.h" +#include "td/utils/port/path.h" +#include "td/utils/Random.h" +#include "td/utils/port/signals.h" +#include "td/utils/port/FileFd.h" +#include "td/utils/overloaded.h" +#include "common/errorlog.h" +#include "tl-utils/common-utils.hpp" +#include "tl/TlObject.h" +#include +#include + +#if TD_DARWIN || TD_LINUX +#include +#endif +#include +#include + +#include + +struct Node { + ton::PrivateKey pk; + ton::PublicKeyHash id; + ton::PublicKey id_full; + ton::adnl::AdnlNodeIdShort adnl_id; + ton::adnl::AdnlNodeIdFull adnl_id_full; + bool can_receive; +}; + +static std::vector root_nodes; +static std::vector slave_nodes; +static std::vector all_nodes; +static td::uint32 total_nodes = 4; +static td::int32 node_slaves_cnt = 3; +static size_t remaining = 0; +static td::Bits256 bcast_hash; + +class Callback : public ton::overlay::Overlays::Callback { + public: + Callback(bool can_receive) : can_receive_(can_receive) { + } + void receive_message(ton::adnl::AdnlNodeIdShort src, ton::overlay::OverlayIdShort overlay_id, + td::BufferSlice data) override { + UNREACHABLE(); + } + void receive_query(ton::adnl::AdnlNodeIdShort src, ton::overlay::OverlayIdShort overlay_id, td::BufferSlice data, + td::Promise promise) override { + UNREACHABLE(); + } + void receive_broadcast(ton::PublicKeyHash src, ton::overlay::OverlayIdShort overlay_id, + td::BufferSlice data) override { + CHECK(can_receive_); + CHECK(td::sha256_bits256(data.as_slice()) == bcast_hash); + CHECK(remaining > 0); + remaining--; + } + + private: + bool can_receive_; +}; + +int main(int argc, char *argv[]) { + SET_VERBOSITY_LEVEL(verbosity_INFO); + td::set_default_failure_signal_handler().ensure(); + + std::string db_root_ = "tmp-dir-test-catchain"; + td::rmrf(db_root_).ignore(); + td::mkdir(db_root_).ensure(); + + td::set_default_failure_signal_handler().ensure(); + + td::actor::ActorOwn keyring; + td::actor::ActorOwn network_manager; + td::actor::ActorOwn adnl; + td::actor::ActorOwn overlay_manager; + + td::actor::Scheduler scheduler({7}); + scheduler.run_in_context([&] { + ton::errorlog::ErrorLog::create(db_root_); + keyring = ton::keyring::Keyring::create(db_root_); + network_manager = td::actor::create_actor("test net"); + adnl = ton::adnl::Adnl::create(db_root_, keyring.get()); + overlay_manager = + ton::overlay::Overlays::create(db_root_, keyring.get(), adnl.get(), td::actor::ActorId{}); + td::actor::send_closure(adnl, &ton::adnl::Adnl::register_network_manager, network_manager.get()); + }); + + td::uint32 att = 0; + for (td::uint32 start = att; att < start + 5; att++) { + LOG(WARNING) << "Test #" << att; + root_nodes.resize(total_nodes); + slave_nodes.resize(total_nodes * node_slaves_cnt); + + auto overlay_id_full = + ton::create_serialize_tl_object(td::BufferSlice(PSTRING() << "TEST" << att)); + ton::overlay::OverlayIdFull overlay_id(overlay_id_full.clone()); + auto overlay_id_short = overlay_id.compute_short_id(); + + ton::overlay::OverlayOptions opts; + opts.max_slaves_in_semiprivate_overlay_ = node_slaves_cnt; + opts.default_permanent_members_flags_ = ton::overlay::OverlayMemberFlags::DoNotReceiveBroadcasts; + + ton::overlay::OverlayPrivacyRules rules( + 20 << 20, ton::overlay::CertificateFlags::AllowFec | ton::overlay::CertificateFlags::Trusted, {}); + + std::vector root_keys; + std::vector root_adnl; + + size_t real_members = 0; + + scheduler.run_in_context([&] { + auto addr = ton::adnl::TestLoopbackNetworkManager::generate_dummy_addr_list(); + + for (auto &n : root_nodes) { + bool receive_bcasts = (real_members == 0) ? true : (td::Random::fast_uint32() & 1); + if (receive_bcasts) { + real_members++; + } + n.can_receive = receive_bcasts; + + auto pk1 = ton::PrivateKey{ton::privkeys::Ed25519::random()}; + auto pub1 = pk1.compute_public_key(); + n.adnl_id_full = ton::adnl::AdnlNodeIdFull{pub1}; + n.adnl_id = ton::adnl::AdnlNodeIdShort{pub1.compute_short_id()}; + td::actor::send_closure(keyring, &ton::keyring::Keyring::add_key, std::move(pk1), true, [](td::Unit) {}); + td::actor::send_closure(adnl, &ton::adnl::Adnl::add_id, ton::adnl::AdnlNodeIdFull{pub1}, addr, + static_cast(0)); + td::actor::send_closure(network_manager, &ton::adnl::TestLoopbackNetworkManager::add_node_id, n.adnl_id, true, + true); + + auto pk2 = ton::PrivateKey{ton::privkeys::Ed25519::random()}; + auto pub2 = pk2.compute_public_key(); + n.id_full = pub2; + n.id = pub2.compute_short_id(); + n.pk = pk2; + td::actor::send_closure(keyring, &ton::keyring::Keyring::add_key, std::move(pk2), true, [](td::Unit) {}); + + LOG(DEBUG) << "created node " << n.adnl_id << " " << n.id; + + all_nodes.push_back(&n); + root_keys.push_back(n.id); + root_adnl.push_back(n.adnl_id); + } + + for (auto &n : slave_nodes) { + bool receive_bcasts = (real_members == 0) ? true : (td::Random::fast_uint32() & 1); + if (receive_bcasts) { + real_members++; + } + n.can_receive = receive_bcasts; + + auto pk1 = ton::PrivateKey{ton::privkeys::Ed25519::random()}; + auto pub1 = pk1.compute_public_key(); + n.adnl_id_full = ton::adnl::AdnlNodeIdFull{pub1}; + n.adnl_id = ton::adnl::AdnlNodeIdShort{pub1.compute_short_id()}; + td::actor::send_closure(keyring, &ton::keyring::Keyring::add_key, std::move(pk1), true, [](td::Unit) {}); + td::actor::send_closure(adnl, &ton::adnl::Adnl::add_id, ton::adnl::AdnlNodeIdFull{pub1}, addr, + static_cast(0)); + td::actor::send_closure(network_manager, &ton::adnl::TestLoopbackNetworkManager::add_node_id, n.adnl_id, true, + true); + + auto pk2 = ton::PrivateKey{ton::privkeys::Ed25519::random()}; + auto pub2 = pk2.compute_public_key(); + n.id_full = pub2; + n.id = pub2.compute_short_id(); + n.pk = pk2; + td::actor::send_closure(keyring, &ton::keyring::Keyring::add_key, std::move(pk2), true, [](td::Unit) {}); + + LOG(DEBUG) << "created node " << n.adnl_id << " " << n.id; + all_nodes.push_back(&n); + } + + for (auto &n1 : all_nodes) { + for (auto &n2 : all_nodes) { + td::actor::send_closure(adnl, &ton::adnl::Adnl::add_peer, n1->adnl_id, n2->adnl_id_full, addr); + } + } + + for (auto &n1 : root_nodes) { + opts.local_overlay_member_flags_ = + (n1.can_receive ? 0 : ton::overlay::OverlayMemberFlags::DoNotReceiveBroadcasts); + td::actor::send_closure(overlay_manager, &ton::overlay::Overlays::create_semiprivate_overlay, n1.adnl_id, + ton::overlay::OverlayIdFull(overlay_id_full.clone()), root_adnl, root_keys, + ton::overlay::OverlayMemberCertificate{}, std::make_unique(n1.can_receive), + rules, "", opts); + } + for (size_t i = 0; i < slave_nodes.size(); i++) { + auto &n1 = slave_nodes[i]; + opts.local_overlay_member_flags_ = + (n1.can_receive ? 0 : ton::overlay::OverlayMemberFlags::DoNotReceiveBroadcasts); + + ton::overlay::OverlayMemberCertificate cert(root_nodes[i / node_slaves_cnt].id_full, 0, i % node_slaves_cnt, + 2000000000, td::BufferSlice()); + auto buf = cert.to_sign_data(n1.adnl_id); + auto dec = root_nodes[i / node_slaves_cnt].pk.create_decryptor().move_as_ok(); + auto signature = dec->sign(buf.as_slice()).move_as_ok(); + cert.set_signature(signature.as_slice()); + auto enc = root_nodes[i / node_slaves_cnt].id_full.create_encryptor().move_as_ok(); + enc->check_signature(cert.to_sign_data(n1.adnl_id), cert.signature()).ensure(); + + td::actor::send_closure(overlay_manager, &ton::overlay::Overlays::create_semiprivate_overlay, n1.adnl_id, + ton::overlay::OverlayIdFull(overlay_id_full.clone()), root_adnl, root_keys, cert, + std::make_unique(n1.can_receive), rules, "", opts); + } + }); + + td::BufferSlice broadcast(1 << 20); + td::Random::secure_bytes(broadcast.as_slice()); + remaining = real_members; + bcast_hash = td::sha256_bits256(broadcast.as_slice()); + + auto t = td::Timestamp::in(20.0); + while (scheduler.run(1)) { + if (t.is_in_past()) { + break; + } + } + + scheduler.run_in_context([&] { + /*td::actor::send_closure(overlay_manager, &ton::overlay::Overlays::get_stats, + [&](td::Result> R) { + if (R.is_ok()) { + auto res = R.move_as_ok(); + for (auto &o : res->overlays_) { + if (o->overlay_id_ == overlay_id_short.bits256_value()) { + LOG(ERROR) << "NODE " << o->adnl_id_ << " nodes=" << o->nodes_.size(); + for (auto &x : o->stats_) { + LOG(ERROR) << "\t" << x->key_ << " " << x->value_; + } + for (auto &x : o->nodes_) { + LOG(ERROR) << "\t\t" << x->adnl_id_; + } + } + } + } + });*/ + td::actor::send_closure(overlay_manager, &ton::overlay::Overlays::send_broadcast_fec_ex, root_nodes[0].adnl_id, + overlay_id_short, root_nodes[0].id, 0, std::move(broadcast)); + }); + + t = td::Timestamp::in(10.0); + while (scheduler.run(1)) { + if (t.is_in_past()) { + break; + } + if (!remaining) { + break; + } + } + + LOG_CHECK(!remaining) << "remaining=" << remaining << " all=" << real_members; + + broadcast = td::BufferSlice(700); + td::Random::secure_bytes(broadcast.as_slice()); + remaining = real_members; + bcast_hash = td::sha256_bits256(broadcast.as_slice()); + scheduler.run_in_context([&] { + td::actor::send_closure(overlay_manager, &ton::overlay::Overlays::send_broadcast_ex, root_nodes[0].adnl_id, + overlay_id_short, root_nodes[0].id, 0, std::move(broadcast)); + }); + + t = td::Timestamp::in(10.0); + while (scheduler.run(1)) { + if (t.is_in_past()) { + break; + } + if (!remaining) { + break; + } + } + + LOG_CHECK(!remaining) << "remaining=" << remaining; + + scheduler.run_in_context([&] { + root_nodes.clear(); + slave_nodes.clear(); + all_nodes.clear(); + }); + } + + for (td::uint32 start = att; att < start + 5; att++) { + LOG(WARNING) << "Test #" << att; + root_nodes.resize(total_nodes); + + auto overlay_id_full = + ton::create_serialize_tl_object(td::BufferSlice(PSTRING() << "TEST" << att)); + ton::overlay::OverlayIdFull overlay_id(overlay_id_full.clone()); + auto overlay_id_short = overlay_id.compute_short_id(); + + ton::overlay::OverlayOptions opts; + + ton::overlay::OverlayPrivacyRules rules( + 20 << 20, ton::overlay::CertificateFlags::AllowFec | ton::overlay::CertificateFlags::Trusted, {}); + + std::vector root_keys; + std::vector root_adnl; + + size_t real_members = 0; + + scheduler.run_in_context([&] { + auto addr = ton::adnl::TestLoopbackNetworkManager::generate_dummy_addr_list(); + + for (auto &n : root_nodes) { + real_members++; + auto pk1 = ton::PrivateKey{ton::privkeys::Ed25519::random()}; + auto pub1 = pk1.compute_public_key(); + n.adnl_id_full = ton::adnl::AdnlNodeIdFull{pub1}; + n.adnl_id = ton::adnl::AdnlNodeIdShort{pub1.compute_short_id()}; + td::actor::send_closure(keyring, &ton::keyring::Keyring::add_key, std::move(pk1), true, [](td::Unit) {}); + td::actor::send_closure(adnl, &ton::adnl::Adnl::add_id, ton::adnl::AdnlNodeIdFull{pub1}, addr, + static_cast(0)); + td::actor::send_closure(network_manager, &ton::adnl::TestLoopbackNetworkManager::add_node_id, n.adnl_id, true, + true); + + auto pk2 = ton::PrivateKey{ton::privkeys::Ed25519::random()}; + auto pub2 = pk2.compute_public_key(); + n.id_full = pub2; + n.id = pub2.compute_short_id(); + n.pk = pk2; + td::actor::send_closure(keyring, &ton::keyring::Keyring::add_key, std::move(pk2), true, [](td::Unit) {}); + + LOG(DEBUG) << "created node " << n.adnl_id << " " << n.id; + + all_nodes.push_back(&n); + root_keys.push_back(n.id); + root_adnl.push_back(n.adnl_id); + } + + for (auto &n1 : all_nodes) { + for (auto &n2 : all_nodes) { + td::actor::send_closure(adnl, &ton::adnl::Adnl::add_peer, n1->adnl_id, n2->adnl_id_full, addr); + } + } + + for (auto &n1 : root_nodes) { + td::actor::send_closure(overlay_manager, &ton::overlay::Overlays::create_private_overlay_ex, n1.adnl_id, + ton::overlay::OverlayIdFull(overlay_id_full.clone()), root_adnl, + std::make_unique(true), rules, "", opts); + } + }); + + auto t = td::Timestamp::in(10.0); + while (scheduler.run(1)) { + if (t.is_in_past()) { + break; + } + } + + td::BufferSlice broadcast(1 << 20); + td::Random::secure_bytes(broadcast.as_slice()); + remaining = real_members; + bcast_hash = td::sha256_bits256(broadcast.as_slice()); + + scheduler.run_in_context([&] { + td::actor::send_closure(overlay_manager, &ton::overlay::Overlays::send_broadcast_fec_ex, root_nodes[0].adnl_id, + overlay_id_short, root_nodes[0].id, 0, std::move(broadcast)); + }); + + t = td::Timestamp::in(10.0); + while (scheduler.run(1)) { + if (t.is_in_past()) { + break; + } + if (!remaining) { + break; + } + } + + LOG_CHECK(!remaining) << "remaining=" << remaining; + + broadcast = td::BufferSlice(700); + td::Random::secure_bytes(broadcast.as_slice()); + remaining = real_members; + bcast_hash = td::sha256_bits256(broadcast.as_slice()); + scheduler.run_in_context([&] { + td::actor::send_closure(overlay_manager, &ton::overlay::Overlays::send_broadcast_ex, root_nodes[0].adnl_id, + overlay_id_short, root_nodes[0].id, 0, std::move(broadcast)); + }); + + t = td::Timestamp::in(10.0); + while (scheduler.run(1)) { + if (t.is_in_past()) { + break; + } + if (!remaining) { + break; + } + } + + LOG_CHECK(!remaining) << "remaining=" << remaining; + + scheduler.run_in_context([&] { + root_nodes.clear(); + slave_nodes.clear(); + all_nodes.clear(); + }); + } + + td::rmrf(db_root_).ensure(); + std::_Exit(0); + return 0; +} diff --git a/test/test-validator-session-state.cpp b/test/test-validator-session-state.cpp index cb5d817f2..a675070ce 100644 --- a/test/test-validator-session-state.cpp +++ b/test/test-validator-session-state.cpp @@ -101,6 +101,10 @@ class Description : public ton::validatorsession::ValidatorSessionDescription { td::uint32 get_max_priority() const override { return opts_.round_candidates - 1; } + td::uint32 get_node_by_priority(td::uint32 round, td::uint32 priority) const override { + CHECK(priority <= get_max_priority()); + return (round + priority) % get_total_nodes(); + } td::uint32 get_unixtime(td::uint64 ts) const override { return static_cast(ts >> 32); } diff --git a/tl-utils/common-utils.hpp b/tl-utils/common-utils.hpp index d61bd79c0..05f8a825f 100644 --- a/tl-utils/common-utils.hpp +++ b/tl-utils/common-utils.hpp @@ -148,6 +148,39 @@ td::Result::value, T>>> } } +template +td::Result::value, T>>> fetch_tl_prefix(td::Slice &data, + bool boxed) { + td::TlParser p(data); + tl_object_ptr R; + if (boxed) { + R = TlFetchBoxed, T::ID>::parse(p); + } else { + R = move_tl_object_as(T::fetch(p)); + } + if (p.get_status().is_ok()) { + data.remove_prefix(data.size() - p.get_left_len()); + return std::move(R); + } else { + return p.get_status(); + } +} + +template +td::Result::value, T>>> fetch_tl_prefix(td::Slice &data, + bool boxed) { + CHECK(boxed); + td::TlParser p(data); + tl_object_ptr R; + R = move_tl_object_as(T::fetch(p)); + if (p.get_status().is_ok()) { + data.remove_prefix(data.size() - p.get_left_len()); + return std::move(R); + } else { + return p.get_status(); + } +} + template [[deprecated]] tl_object_ptr clone_tl_object(const tl_object_ptr &obj) { auto B = serialize_tl_object(obj, true); diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index bf919b0fd..bc36827d9 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -209,10 +209,15 @@ dht.query node:dht.node = True; ---types--- overlay.node.toSign id:adnl.id.short overlay:int256 version:int = overlay.node.ToSign; +overlay.node.toSignEx id:adnl.id.short overlay:int256 flags:int version:int = overlay.node.ToSign; overlay.node id:PublicKey overlay:int256 version:int signature:bytes = overlay.Node; +overlay.nodeV2 id:PublicKey overlay:int256 flags:int version:int signature:bytes certificate:overlay.MemberCertificate = overlay.NodeV2; overlay.nodes nodes:(vector overlay.node) = overlay.Nodes; +overlay.nodesV2 nodes:(vector overlay.NodeV2) = overlay.NodesV2; +overlay.messageExtra flags:# certificate:flags.0?overlay.MemberCertificate = overlay.MessageExtra; overlay.message overlay:int256 = overlay.Message; +overlay.messageWithExtra overlay:int256 extra:overlay.messageExtra = overlay.Message; //overlay.randomPeers peers:(vector adnl.node) = overlay.RandomPeers; overlay.broadcastList hashes:(vector int256) = overlay.BroadcastList; @@ -225,6 +230,9 @@ overlay.broadcastFec.partId broadcast_hash:int256 data_hash:int256 seqno:int = o overlay.broadcast.toSign hash:int256 date:int = overlay.broadcast.ToSign; +overlay.memberCertificateId node:adnl.id.short flags:int slot:int expire_at:int = overlay.MemberCertificateId; +overlay.memberCertificate issued_by:PublicKey flags:int slot:int expire_at:int signature:bytes = overlay.MemberCertificate; +overlay.emptyMemberCertificate = overlay.MemberCertificate; overlay.certificate issued_by:PublicKey expire_at:int max_size:int signature:bytes = overlay.Certificate; overlay.certificateV2 issued_by:PublicKey expire_at:int max_size:int flags:int signature:bytes = overlay.Certificate; overlay.emptyCertificate = overlay.Certificate; @@ -242,13 +250,16 @@ overlay.broadcastNotFound = overlay.Broadcast; ---functions--- overlay.getRandomPeers peers:overlay.nodes = overlay.Nodes; +overlay.getRandomPeersV2 peers:overlay.NodesV2 = overlay.NodesV2; overlay.query overlay:int256 = True; +overlay.queryWithExtra overlay:int256 extra:overlay.messageExtra = True; overlay.getBroadcast hash:int256 = overlay.Broadcast; overlay.getBroadcastList list:overlay.broadcastList = overlay.BroadcastList; ---types--- +overlay.db.nodesV2 nodes:overlay.nodesV2 = overlay.db.Nodes; overlay.db.nodes nodes:overlay.nodes = overlay.db.Nodes; overlay.db.key.nodes local_id:int256 overlay:int256 = overlay.db.Key; @@ -366,6 +377,7 @@ tonNode.blockSignature who:int256 signature:bytes = tonNode.BlockSignature; tonNode.blockId workchain:int shard:long seqno:int = tonNode.BlockId; tonNode.blockIdExt workchain:int shard:long seqno:int root_hash:int256 file_hash:int256 = tonNode.BlockIdExt; tonNode.zeroStateIdExt workchain:int root_hash:int256 file_hash:int256 = tonNode.ZeroStateIdExt; +tonNode.shardId workchain:int shard:long = tonNode.ShardId; tonNode.blockDescriptionEmpty = tonNode.BlockDescription; tonNode.blockDescription id:tonNode.blockIdExt = tonNode.BlockDescription; @@ -502,6 +514,7 @@ db.filedb.key.proof block_id:tonNode.blockIdExt = db.filedb.Key; db.filedb.key.proofLink block_id:tonNode.blockIdExt = db.filedb.Key; db.filedb.key.signatures block_id:tonNode.blockIdExt = db.filedb.Key; db.filedb.key.candidate id:db.candidate.id = db.filedb.Key; +db.filedb.key.candidateRef id:tonNode.blockIdExt = db.filedb.Key; db.filedb.key.blockInfo block_id:tonNode.blockIdExt = db.filedb.Key; db.filedb.value key:db.filedb.Key prev:int256 next:int256 file_hash:int256 = db.filedb.Value; @@ -649,9 +662,10 @@ engine.validator.proposalVote perm_key:int256 to_send:bytes = engine.validator.P engine.validator.dhtServerStatus id:int256 status:int = engine.validator.DhtServerStatus; engine.validator.dhtServersStatus servers:(vector engine.validator.dhtServerStatus) = engine.validator.DhtServersStatus; -engine.validator.overlayStatsNode adnl_id:int256 ip_addr:string bdcst_errors:int fec_bdcst_errors:int last_in_query:int last_out_query:int t_out_bytes:int t_in_bytes:int t_out_pckts:int t_in_pckts:int = engine.validator.OverlayStatsNode; +engine.validator.overlayStatsNode adnl_id:int256 ip_addr:string is_neighbour:Bool is_alive:Bool node_flags:int + bdcst_errors:int fec_bdcst_errors:int last_in_query:int last_out_query:int t_out_bytes:int t_in_bytes:int t_out_pckts:int t_in_pckts:int = engine.validator.OverlayStatsNode; -engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_id:int256 scope:string nodes:(vector engine.validator.overlayStatsNode) stats:(vector engine.validator.oneStat) = engine.validator.OverlayStats; +engine.validator.overlayStats overlay_id:int256 overlay_id_full:PublicKey adnl_id:int256 scope:string nodes:(vector engine.validator.overlayStatsNode) stats:(vector engine.validator.oneStat) extra:string = engine.validator.OverlayStats; engine.validator.overlaysStats overlays:(vector engine.validator.overlayStats) = engine.validator.OverlaysStats; engine.validator.onePerfTimerStat time:int min:double avg:double max:double = engine.validator.OnePerfTimerStat; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 337dd071e3f746cd822236e2dcb63c194b379de4..777402a89204e21840c2a2475f894654f49efe05 100644 GIT binary patch delta 1776 zcmZ`(ZA@Eb6u!6Iy0%$q-EQrLQr2cHw9*u8h%#3YoC{lI%RZ2WNgQQ;X&c&J)^-~; zCZaKdaYor$(wZ4pFlO5ild(I6YFL;O|2Q-8BWj``1dSHQ%_=a4 zoRKNtYkLDOTbs0Ide=)%xMH;n8qTY&{_N`-O$C3NB%*<|cRz{sk;HD2NJaW1Cju!l z)>7&4{W9XQ{jbNqAS{$|tbLR9N7I^9G8qVwu~x3)!{wj6tjfM;D3u66fVM%8-Lv)5 zM;B(w6^{dakaYRrPFV$v+C6O3(tKqR74MQ=zOk#EaNW+(mScZ8e={qA;)A7oXfxP2 z-R+5yt;$VY4D)vKMH^rcm;8xCWur1wjYl+m= z^7pGP=(WBwS#c{D?Iz@#qVOA0jBsdt@8(GIc$`E+;XeLU0>+azy-u_QqLI@CzYGTK z-|_xvAO!JUl|pvBiIx+8KM|G+6IwsGYH@d;r9LmpIHr~{MWM_wrxvdV6tOW*5sDh8WQaD@w=anI zVttwp22M6xGt(`!c|(=840^%hw1dHM%M7>E4Lk9)zToEVz6nk#I7XyFJQ_)brJ)lY z(I1J>z$^LMZn1s(5X^m$STu4|iJMK(oL%&?vqB?Cr* zKR?Hn>;i0jmybdl4F1WTJ7$K_yd$%EfY!mmNiMT$qN}iPnOV0;mg^eJJqTM0zn354 zN_Giii@OAEU>>bH_W~>BF1)taooSE{_e2Y>C|G@ylRG_{_ro`r_4T4rDFH`%CG_HJ79}ZW^roaewr5D>5&;&5 zivXp>$O(kAH@FtAWZ5Otmk1v_ zNJv5?Osoz$!4}i->usy_Y^7wSo#>J2X{UobR=)W6{3FT(PM#u(^qLfRe)q>yMIo{e M65>>@oz6P{16#p$U;qFB delta 594 zcmX@|gmuRyR^CUm^{p77fNLY~X--k~jHO3;QE;q3C*C|J40SwO;*-z>=hTeLZ1=?2ls9D7ArK+!Wt~{>7g$V*8l@#tKCM1O4mY diff --git a/ton/ton-tl.hpp b/ton/ton-tl.hpp index 6d676ea8d..1bc9a28c8 100644 --- a/ton/ton-tl.hpp +++ b/ton/ton-tl.hpp @@ -19,7 +19,8 @@ #pragma once #include "ton-types.h" -#include "auto/tl/ton_api.h" +#include "auto/tl/ton_api.hpp" +#include "td/utils/overloaded.h" namespace ton { @@ -53,4 +54,12 @@ inline ZeroStateIdExt create_zero_state_id(tl_object_ptrworkchain_, B->root_hash_, B->file_hash_}; } +inline ShardIdFull create_shard_id(const tl_object_ptr &s) { + return ShardIdFull{s->workchain_, static_cast(s->shard_)}; +} + +inline tl_object_ptr create_tl_shard_id(const ShardIdFull &s) { + return create_tl_object(s.workchain, s.shard); +} + } // namespace ton diff --git a/ton/ton-types.h b/ton/ton-types.h index 915682655..efdb795d1 100644 --- a/ton/ton-types.h +++ b/ton/ton-types.h @@ -51,6 +51,8 @@ using ValidatorSessionId = td::Bits256; constexpr WorkchainId masterchainId = -1, basechainId = 0, workchainInvalid = 0x80000000; constexpr ShardId shardIdAll = (1ULL << 63); +constexpr int max_shard_pfx_len = 60; + enum GlobalCapabilities { capIhrEnabled = 1, capCreateStatsEnabled = 2, diff --git a/validator-session/candidate-serializer.cpp b/validator-session/candidate-serializer.cpp index a85c4b72e..4442504ed 100644 --- a/validator-session/candidate-serializer.cpp +++ b/validator-session/candidate-serializer.cpp @@ -22,27 +22,15 @@ namespace ton::validatorsession { -td::Result serialize_candidate(const tl_object_ptr &block, +td::Result serialize_candidate(const tl_object_ptr& block, bool compression_enabled) { if (!compression_enabled) { return serialize_tl_object(block, true); } - vm::BagOfCells boc1, boc2; - TRY_STATUS(boc1.deserialize(block->data_)); - if (boc1.get_root_count() != 1) { - return td::Status::Error("block candidate should have exactly one root"); - } - std::vector> roots = {boc1.get_root_cell()}; - TRY_STATUS(boc2.deserialize(block->collated_data_)); - for (int i = 0; i < boc2.get_root_count(); ++i) { - roots.push_back(boc2.get_root_cell(i)); - } - TRY_RESULT(data, vm::std_boc_serialize_multi(std::move(roots), 2)); - td::BufferSlice compressed = td::lz4_compress(data); - LOG(VALIDATOR_SESSION_DEBUG) << "Compressing block candidate: " << block->data_.size() + block->collated_data_.size() - << " -> " << compressed.size(); + size_t decompressed_size; + TRY_RESULT(compressed, compress_candidate_data(block->data_, block->collated_data_, decompressed_size)) return create_serialize_tl_object( - 0, block->src_, block->round_, block->root_hash_, (int)data.size(), std::move(compressed)); + 0, block->src_, block->round_, block->root_hash_, (int)decompressed_size, std::move(compressed)); } td::Result> deserialize_candidate(td::Slice data, @@ -55,8 +43,34 @@ td::Result> deserialize_candi if (f->decompressed_size_ > max_decompressed_data_size) { return td::Status::Error("decompressed size is too big"); } - TRY_RESULT(decompressed, td::lz4_decompress(f->data_, f->decompressed_size_)); - if (decompressed.size() != (size_t)f->decompressed_size_) { + TRY_RESULT(p, decompress_candidate_data(f->data_, f->decompressed_size_)); + return create_tl_object(f->src_, f->round_, f->root_hash_, std::move(p.first), + std::move(p.second)); +} + +td::Result compress_candidate_data(td::Slice block, td::Slice collated_data, + size_t& decompressed_size) { + vm::BagOfCells boc1, boc2; + TRY_STATUS(boc1.deserialize(block)); + if (boc1.get_root_count() != 1) { + return td::Status::Error("block candidate should have exactly one root"); + } + std::vector> roots = {boc1.get_root_cell()}; + TRY_STATUS(boc2.deserialize(collated_data)); + for (int i = 0; i < boc2.get_root_count(); ++i) { + roots.push_back(boc2.get_root_cell(i)); + } + TRY_RESULT(data, vm::std_boc_serialize_multi(std::move(roots), 2)); + decompressed_size = data.size(); + td::BufferSlice compressed = td::lz4_compress(data); + LOG(DEBUG) << "Compressing block candidate: " << block.size() + collated_data.size() << " -> " << compressed.size(); + return compressed; +} + +td::Result> decompress_candidate_data(td::Slice compressed, + int decompressed_size) { + TRY_RESULT(decompressed, td::lz4_decompress(compressed, decompressed_size)); + if (decompressed.size() != (size_t)decompressed_size) { return td::Status::Error("decompressed size mismatch"); } TRY_RESULT(roots, vm::std_boc_deserialize_multi(decompressed)); @@ -66,10 +80,9 @@ td::Result> deserialize_candi TRY_RESULT(block_data, vm::std_boc_serialize(roots[0], 31)); roots.erase(roots.begin()); TRY_RESULT(collated_data, vm::std_boc_serialize_multi(std::move(roots), 31)); - LOG(VALIDATOR_SESSION_DEBUG) << "Decompressing block candidate: " << f->data_.size() << " -> " - << block_data.size() + collated_data.size(); - return create_tl_object(f->src_, f->round_, f->root_hash_, std::move(block_data), - std::move(collated_data)); + LOG(DEBUG) << "Decompressing block candidate: " << compressed.size() << " -> " + << block_data.size() + collated_data.size(); + return std::make_pair(std::move(block_data), std::move(collated_data)); } } // namespace ton::validatorsession diff --git a/validator-session/candidate-serializer.h b/validator-session/candidate-serializer.h index 030a412c0..e88376cd7 100644 --- a/validator-session/candidate-serializer.h +++ b/validator-session/candidate-serializer.h @@ -20,10 +20,15 @@ namespace ton::validatorsession { -td::Result serialize_candidate(const tl_object_ptr &block, +td::Result serialize_candidate(const tl_object_ptr& block, bool compression_enabled); td::Result> deserialize_candidate(td::Slice data, bool compression_enabled, int max_decompressed_data_size); +td::Result compress_candidate_data(td::Slice block, td::Slice collated_data, + size_t& decompressed_size); +td::Result> decompress_candidate_data(td::Slice compressed, + int decompressed_size); + } // namespace ton::validatorsession diff --git a/validator-session/validator-session-description.cpp b/validator-session/validator-session-description.cpp index 34dc90212..c747e26e7 100644 --- a/validator-session/validator-session-description.cpp +++ b/validator-session/validator-session-description.cpp @@ -72,6 +72,11 @@ td::uint32 ValidatorSessionDescriptionImpl::get_max_priority() const { return opts_.round_candidates - 1; } +td::uint32 ValidatorSessionDescriptionImpl::get_node_by_priority(td::uint32 round, td::uint32 priority) const { + CHECK(priority <= get_max_priority()); + return (round + priority) % get_total_nodes(); +} + ValidatorSessionCandidateId ValidatorSessionDescriptionImpl::candidate_id( td::uint32 src_idx, ValidatorSessionRootHash root_hash, ValidatorSessionFileHash file_hash, ValidatorSessionCollatedDataFileHash collated_data_file_hash) const { diff --git a/validator-session/validator-session-description.h b/validator-session/validator-session-description.h index 60e006d08..ac5043125 100644 --- a/validator-session/validator-session-description.h +++ b/validator-session/validator-session-description.h @@ -85,6 +85,7 @@ class ValidatorSessionDescription { virtual ValidatorWeight get_total_weight() const = 0; virtual td::int32 get_node_priority(td::uint32 src_idx, td::uint32 round) const = 0; virtual td::uint32 get_max_priority() const = 0; + virtual td::uint32 get_node_by_priority(td::uint32 round, td::uint32 priority) const = 0; virtual td::uint32 get_unixtime(td::uint64 t) const = 0; virtual td::uint32 get_attempt_seqno(td::uint64 t) const = 0; virtual td::uint32 get_self_idx() const = 0; diff --git a/validator-session/validator-session-description.hpp b/validator-session/validator-session-description.hpp index 5e09c694a..0fbb4edeb 100644 --- a/validator-session/validator-session-description.hpp +++ b/validator-session/validator-session-description.hpp @@ -114,6 +114,7 @@ class ValidatorSessionDescriptionImpl : public ValidatorSessionDescription { } td::int32 get_node_priority(td::uint32 src_idx, td::uint32 round) const override; td::uint32 get_max_priority() const override; + td::uint32 get_node_by_priority(td::uint32 round, td::uint32 priority) const override; td::uint32 get_unixtime(td::uint64 ts) const override { return static_cast(ts >> 32); } diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index be5443785..07fb5994c 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -21,6 +21,7 @@ #include "td/utils/crypto.h" #include "candidate-serializer.h" #include "td/utils/overloaded.h" +#include "ton/ton-tl.hpp" namespace ton { diff --git a/validator/db/fileref.cpp b/validator/db/fileref.cpp index 2e2a3b6b6..378e382c9 100644 --- a/validator/db/fileref.cpp +++ b/validator/db/fileref.cpp @@ -213,6 +213,28 @@ std::string CandidateShort::filename_short() const { return PSTRING() << "candidate_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" << hash().to_hex(); } +CandidateRefShort CandidateRef::shortref() const { + return CandidateRefShort{block_id.id, hash()}; +} + +std::string CandidateRef::filename() const { + return PSTRING() << "candidateref_" << block_id.to_str(); +} + +std::string CandidateRef::filename_short() const { + char s[33]; + sprintf(s, "%llx", static_cast(block_id.id.shard)); + return PSTRING() << "candidateref_" << block_id.id.workchain << "_" << s << "_" << block_id.id.seqno << "_" + << hash().to_hex(); +} + +std::string CandidateRefShort::filename_short() const { + char s[33]; + sprintf(s, "%llx", static_cast(block_id.shard)); + return PSTRING() << "candidateref_" << block_id.workchain << "_" << s << "_" << block_id.seqno << "_" + << hash().to_hex(); +} + BlockInfoShort BlockInfo::shortref() const { return BlockInfoShort{block_id.id, hash()}; } @@ -259,6 +281,9 @@ FileReference::FileReference(tl_object_ptr key) { ref_ = fileref::Candidate{PublicKey{key.id_->source_}, create_block_id(key.id_->id_), key.id_->collated_data_file_hash_}; }, + [&](const ton_api::db_filedb_key_candidateRef& key) { + ref_ = fileref::CandidateRef{create_block_id(key.id_)}; + }, [&](const ton_api::db_filedb_key_blockInfo& key) { ref_ = fileref::BlockInfo{create_block_id(key.block_id_)}; })); diff --git a/validator/db/fileref.hpp b/validator/db/fileref.hpp index 14424a653..6710cc063 100644 --- a/validator/db/fileref.hpp +++ b/validator/db/fileref.hpp @@ -278,6 +278,38 @@ class Candidate { FileHash collated_data_file_hash; }; +class CandidateRefShort { + public: + FileHash hash() const { + return hashv; + } + ShardIdFull shard() const { + return block_id.shard_full(); + } + std::string filename_short() const; + + BlockId block_id; + FileHash hashv; +}; + +class CandidateRef { + public: + tl_object_ptr tl() const { + return create_tl_object(create_tl_block_id(block_id)); + } + FileHash hash() const { + return create_hash_tl_object(create_tl_block_id(block_id)); + } + ShardIdFull shard() const { + return block_id.shard_full(); + } + CandidateRefShort shortref() const; + std::string filename() const; + std::string filename_short() const; + + BlockIdExt block_id; +}; + class BlockInfoShort { public: FileHash hash() const { @@ -316,7 +348,7 @@ class FileReferenceShort { private: td::Variant + fileref::CandidateShort, fileref::CandidateRefShort, fileref::BlockInfoShort> ref_; public: @@ -340,7 +372,8 @@ class FileReferenceShort { class FileReference { private: td::Variant + fileref::Proof, fileref::ProofLink, fileref::Signatures, fileref::Candidate, fileref::CandidateRef, + fileref::BlockInfo> ref_; public: diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 93dcfc91f..891d551f8 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -177,21 +177,21 @@ void RootDb::get_block_proof_link(ConstBlockHandle handle, td::Promise promise) { + auto source = PublicKey{pubkeys::Ed25519{candidate.pubkey.as_bits256()}}; auto obj = create_serialize_tl_object( - PublicKey{pubkeys::Ed25519{candidate.pubkey.as_bits256()}}.tl(), create_tl_block_id(candidate.id), - std::move(candidate.data), std::move(candidate.collated_data)); - - auto P = td::PromiseCreator::lambda([promise = std::move(promise)](td::Result R) mutable { - if (R.is_error()) { - promise.set_error(R.move_as_error()); - } else { - promise.set_value(td::Unit()); - } - }); + source.tl(), create_tl_block_id(candidate.id), std::move(candidate.data), std::move(candidate.collated_data)); + auto P = td::PromiseCreator::lambda( + [archive_db = archive_db_.get(), promise = std::move(promise), block_id = candidate.id, source, + collated_file_hash = candidate.collated_file_hash](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, _, std::move(R)); + td::actor::send_closure(archive_db, &ArchiveManager::add_temp_file_short, fileref::CandidateRef{block_id}, + create_serialize_tl_object( + source.tl(), create_tl_block_id(block_id), collated_file_hash), + std::move(promise)); + }); td::actor::send_closure(archive_db_, &ArchiveManager::add_temp_file_short, - fileref::Candidate{PublicKey{pubkeys::Ed25519{candidate.pubkey.as_bits256()}}, candidate.id, - candidate.collated_file_hash}, - std::move(obj), std::move(P)); + fileref::Candidate{source, candidate.id, candidate.collated_file_hash}, std::move(obj), + std::move(P)); } void RootDb::get_block_candidate(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash, @@ -215,6 +215,17 @@ void RootDb::get_block_candidate(PublicKey source, BlockIdExt id, FileHash colla fileref::Candidate{source, id, collated_data_file_hash}, std::move(P)); } +void RootDb::get_block_candidate_by_block_id(BlockIdExt id, td::Promise promise) { + td::actor::send_closure( + archive_db_, &ArchiveManager::get_temp_file_short, fileref::CandidateRef{id}, + [SelfId = actor_id(this), promise = std::move(promise)](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, data, std::move(R)); + TRY_RESULT_PROMISE(promise, f, fetch_tl_object(data, true)); + td::actor::send_closure(SelfId, &RootDb::get_block_candidate, PublicKey{f->source_}, create_block_id(f->id_), + f->collated_data_file_hash_, std::move(promise)); + }); +} + void RootDb::store_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) { if (handle->moved_to_archive()) { diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index 45044e4f8..755ff2578 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -58,6 +58,7 @@ class RootDb : public Db { void store_block_candidate(BlockCandidate candidate, td::Promise promise) override; void get_block_candidate(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash, td::Promise promise) override; + void get_block_candidate_by_block_id(BlockIdExt id, td::Promise promise) override; void store_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) override; diff --git a/validator/db/statedb.hpp b/validator/db/statedb.hpp index 75382d61c..a7a004522 100644 --- a/validator/db/statedb.hpp +++ b/validator/db/statedb.hpp @@ -50,9 +50,6 @@ class StateDb : public td::actor::Actor { void update_hardforks(std::vector blocks, td::Promise promise); void get_hardforks(td::Promise> promise); - void update_db_version(td::uint32 version, td::Promise promise); - void get_db_version(td::Promise promise); - StateDb(td::actor::ActorId root_db, std::string path); void start_up() override; diff --git a/validator/fabric.h b/validator/fabric.h index 949a6c9ff..fabdf8e3c 100644 --- a/validator/fabric.h +++ b/validator/fabric.h @@ -74,12 +74,12 @@ void run_check_proof_query(BlockIdExt id, td::Ref proof, td::actor::Actor td::Ref rel_key_block_proof, bool skip_check_signatures = false); void run_check_proof_link_query(BlockIdExt id, td::Ref proof, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise); -void run_validate_query(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_masterchain_block_id, - std::vector prev, BlockCandidate candidate, td::Ref validator_set, +void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, + BlockCandidate candidate, td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, bool is_fake = false); -void run_collate_query(ShardIdFull shard, td::uint32 min_ts, const BlockIdExt& min_masterchain_block_id, - std::vector prev, Ed25519_PublicKey local_id, td::Ref validator_set, +void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, + Ed25519_PublicKey creator, td::Ref validator_set, td::Ref collator_opts, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise); void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, diff --git a/validator/impl/CMakeLists.txt b/validator/impl/CMakeLists.txt index 42dded311..9df6725d6 100644 --- a/validator/impl/CMakeLists.txt +++ b/validator/impl/CMakeLists.txt @@ -27,7 +27,6 @@ set(TON_VALIDATOR_SOURCE block.hpp candidates-buffer.hpp check-proof.hpp - collate-query-impl.h collator-impl.h collator.h config.hpp diff --git a/validator/impl/accept-block.cpp b/validator/impl/accept-block.cpp index 3da1167aa..cfdeb0523 100644 --- a/validator/impl/accept-block.cpp +++ b/validator/impl/accept-block.cpp @@ -412,8 +412,37 @@ void AcceptBlockQuery::got_block_handle(BlockHandle handle) { (is_masterchain() ? handle_->inited_proof() && handle_->is_applied() && handle_->inited_is_key_block() : handle_->inited_proof_link())) { finish_query(); + return; + } + if (data_.is_null()) { + td::actor::send_closure(manager_, &ValidatorManager::get_candidate_data_by_block_id_from_db, id_, [SelfId = actor_id(this)](td::Result R) { + if (R.is_ok()) { + td::actor::send_closure(SelfId, &AcceptBlockQuery::got_block_candidate_data, R.move_as_ok()); + } else { + td::actor::send_closure(SelfId, &AcceptBlockQuery::got_block_handle_cont); + } + }); + } else { + got_block_handle_cont(); + } +} + +void AcceptBlockQuery::got_block_candidate_data(td::BufferSlice data) { + auto r_block = create_block(id_, std::move(data)); + if (r_block.is_error()) { + fatal_error("invalid block candidate data in db: " + r_block.error().to_string()); return; } + data_ = r_block.move_as_ok(); + VLOG(VALIDATOR_DEBUG) << "got block candidate data from db"; + if (data_.not_null() && !precheck_header()) { + fatal_error("invalid block header in AcceptBlock"); + return; + } + got_block_handle_cont(); +} + +void AcceptBlockQuery::got_block_handle_cont() { if (data_.not_null() && !handle_->received()) { td::actor::send_closure( manager_, &ValidatorManager::set_block_data, handle_, data_, [SelfId = actor_id(this)](td::Result R) { diff --git a/validator/impl/accept-block.hpp b/validator/impl/accept-block.hpp index 2600b2581..95b5f2818 100644 --- a/validator/impl/accept-block.hpp +++ b/validator/impl/accept-block.hpp @@ -71,6 +71,8 @@ class AcceptBlockQuery : public td::actor::Actor { void written_block_data(); void written_block_signatures(); void got_block_handle(BlockHandle handle); + void got_block_candidate_data(td::BufferSlice data); + void got_block_handle_cont(); void written_block_info(); void got_block_data(td::Ref data); void got_prev_state(td::Ref state); diff --git a/validator/impl/collate-query-impl.h b/validator/impl/collate-query-impl.h deleted file mode 100644 index 838326468..000000000 --- a/validator/impl/collate-query-impl.h +++ /dev/null @@ -1,63 +0,0 @@ -/* - This file is part of TON Blockchain Library. - - TON Blockchain Library is free software: you can redistribute it and/or modify - it under the terms of the GNU Lesser General Public License as published by - the Free Software Foundation, either version 2 of the License, or - (at your option) any later version. - - TON Blockchain Library is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU Lesser General Public License for more details. - - You should have received a copy of the GNU Lesser General Public License - along with TON Blockchain Library. If not, see . - - Copyright 2017-2020 Telegram Systems LLP -*/ -#pragma once - -#include "validator/interfaces/validator-manager.h" - -namespace ton { - -namespace validator { - -class CollateQuery : public td::actor::Actor { - public: - CollateQuery(ShardIdFull shard, td::uint32 min_ts, BlockIdExt min_masterchain_block_id, std::vector prev, - td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise); - CollateQuery(ShardIdFull shard, td::uint32 min_ts, BlockIdExt min_masterchain_block_id, ZeroStateIdExt zero_state_id, - td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise); - - void alarm() override; - - void abort_query(td::Status reason); - void finish_query(); - - void start_up() override; - void got_prev_state(td::Ref state); - void written_block_data(); - void written_block_collated_data(); - - private: - ShardIdFull shard_; - UnixTime min_ts_; - BlockIdExt min_masterchain_block_id_; - std::vector prev_; - ZeroStateIdExt zero_state_id_; - td::Ref validator_set_; - td::actor::ActorId manager_; - td::Timestamp timeout_; - td::Promise promise_; - - BlockCandidate candidate_; - UnixTime ts_; -}; - -} // namespace validator - -} // namespace ton diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index b8d9e56d3..708278545 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -65,7 +65,6 @@ class Collator final : public td::actor::Actor { bool libraries_changed_{false}; bool prev_key_block_exists_{false}; bool is_hardfork_{false}; - UnixTime min_ts; BlockIdExt min_mc_block_id; std::vector prev_blocks; std::vector> prev_states; @@ -89,10 +88,9 @@ class Collator final : public td::actor::Actor { static constexpr bool shard_splitting_enabled = true; public: - Collator(ShardIdFull shard, bool is_hardfork, td::uint32 min_ts, BlockIdExt min_masterchain_block_id, - std::vector prev, Ref validator_set, Ed25519_PublicKey collator_id, - Ref collator_opts, td::actor::ActorId manager, td::Timestamp timeout, - td::Promise promise); + Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, std::vector prev, + Ref validator_set, Ed25519_PublicKey collator_id, Ref collator_opts, + td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise); ~Collator() override = default; bool is_busy() const { return busy_; diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index f465c0f55..c0fd2fae1 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -1,7 +1,7 @@ /* - This file is part of TON Blockchain Library. + This file is part of TON Blockchain Library. - TON Blockchain Library is free software: you can redistribute it and/or modify + TON Blockchain Library is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License as published by the Free Software Foundation, either version 2 of the License, or (at your option) any later version. @@ -66,7 +66,6 @@ static inline bool dbg(int c) { * * @param shard The shard of the new block. * @param is_hardfork A boolean indicating whether the new block is a hardfork. - * @param min_ts The minimum UnixTime for the new block. * @param min_masterchain_block_id The the minimum reference masterchain block. * @param prev A vector of BlockIdExt representing the previous blocks. * @param validator_set A reference to the ValidatorSet. @@ -76,13 +75,12 @@ static inline bool dbg(int c) { * @param timeout The timeout for the collator. * @param promise The promise to return the result. */ -Collator::Collator(ShardIdFull shard, bool is_hardfork, UnixTime min_ts, BlockIdExt min_masterchain_block_id, - std::vector prev, Ref validator_set, Ed25519_PublicKey collator_id, +Collator::Collator(ShardIdFull shard, bool is_hardfork, BlockIdExt min_masterchain_block_id, + std::vector prev, td::Ref validator_set, Ed25519_PublicKey collator_id, Ref collator_opts, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise) : shard_(shard) , is_hardfork_(is_hardfork) - , min_ts(min_ts) , min_mc_block_id{min_masterchain_block_id} , prev_blocks(std::move(prev)) , created_by_(collator_id) diff --git a/validator/impl/collator.h b/validator/impl/collator.h index c92fa80e3..5acc20e8d 100644 --- a/validator/impl/collator.h +++ b/validator/impl/collator.h @@ -24,26 +24,7 @@ #include "vm/cells.h" namespace ton { -using td::Ref; extern int collator_settings; // +1 = force want_split, +2 = force want_merge -class Collator : public td::actor::Actor { - protected: - Collator() = default; - - public: - virtual ~Collator() = default; - static td::actor::ActorOwn create_collator( - td::actor::ActorId block_db, - ShardIdFull shard /* , td::actor::ActorId validator_manager */); - virtual void generate_block_candidate(ShardIdFull shard, td::Promise promise) = 0; - virtual td::Result register_external_message_cell(Ref ext_msg) = 0; - virtual td::Result register_external_message(td::Slice ext_msg_boc) = 0; - virtual td::Result register_ihr_message_cell(Ref ihr_msg) = 0; - virtual td::Result register_ihr_message(td::Slice ihr_msg_boc) = 0; - virtual td::Result register_shard_signatures_cell(Ref shard_blk_signatures) = 0; - virtual td::Result register_shard_signatures(td::Slice shard_blk_signatures_boc) = 0; -}; - } // namespace ton diff --git a/validator/impl/fabric.cpp b/validator/impl/fabric.cpp index d69492393..bfc25f6ed 100644 --- a/validator/impl/fabric.cpp +++ b/validator/impl/fabric.cpp @@ -133,9 +133,9 @@ void run_accept_block_query(BlockIdExt id, td::Ref data, std::vector< td::Ref validator_set, td::Ref signatures, td::Ref approve_signatures, bool send_broadcast, td::actor::ActorId manager, td::Promise promise) { - td::actor::create_actor("accept", id, std::move(data), prev, std::move(validator_set), - std::move(signatures), std::move(approve_signatures), send_broadcast, - manager, std::move(promise)) + td::actor::create_actor(PSTRING() << "accept" << id.id.to_str(), id, std::move(data), prev, + std::move(validator_set), std::move(signatures), + std::move(approve_signatures), send_broadcast, manager, std::move(promise)) .release(); } @@ -192,7 +192,7 @@ void run_check_proof_link_query(BlockIdExt id, td::Ref proof, td::act .release(); } -void run_validate_query(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_masterchain_block_id, +void run_validate_query(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, BlockCandidate candidate, td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, bool is_fake) { @@ -205,14 +205,14 @@ void run_validate_query(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_maste static std::atomic idx; td::actor::create_actor(PSTRING() << (is_fake ? "fakevalidate" : "validateblock") << shard.to_str() << ":" << (seqno + 1) << "#" << idx.fetch_add(1), - shard, min_ts, min_masterchain_block_id, std::move(prev), std::move(candidate), + shard, min_masterchain_block_id, std::move(prev), std::move(candidate), std::move(validator_set), std::move(manager), timeout, std::move(promise), is_fake) .release(); } -void run_collate_query(ShardIdFull shard, td::uint32 min_ts, const BlockIdExt& min_masterchain_block_id, - std::vector prev, Ed25519_PublicKey collator_id, td::Ref validator_set, +void run_collate_query(ShardIdFull shard, const BlockIdExt& min_masterchain_block_id, std::vector prev, + Ed25519_PublicKey creator, td::Ref validator_set, td::Ref collator_opts, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise) { BlockSeqno seqno = 0; @@ -222,9 +222,8 @@ void run_collate_query(ShardIdFull shard, td::uint32 min_ts, const BlockIdExt& m } } td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, false, - min_ts, min_masterchain_block_id, std::move(prev), std::move(validator_set), - collator_id, std::move(collator_opts), std::move(manager), timeout, - std::move(promise)) + min_masterchain_block_id, std::move(prev), std::move(validator_set), creator, + std::move(collator_opts), std::move(manager), timeout, std::move(promise)) .release(); } @@ -237,7 +236,7 @@ void run_collate_hardfork(ShardIdFull shard, const BlockIdExt& min_masterchain_b seqno = p.seqno(); } } - td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, true, 0, + td::actor::create_actor(PSTRING() << "collate" << shard.to_str() << ":" << (seqno + 1), shard, true, min_masterchain_block_id, std::move(prev), td::Ref{}, Ed25519_PublicKey{Bits256::zero()}, td::Ref{true}, std::move(manager), timeout, std::move(promise)) diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index e899926a0..917ddef24 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -373,7 +373,8 @@ td::Status MasterchainStateQ::mc_init() { td::Status MasterchainStateQ::mc_reinit() { auto res = block::ConfigInfo::extract_config( root_cell(), block::ConfigInfo::needStateRoot | block::ConfigInfo::needValidatorSet | - block::ConfigInfo::needShardHashes | block::ConfigInfo::needPrevBlocks); + block::ConfigInfo::needShardHashes | block::ConfigInfo::needPrevBlocks | + block::ConfigInfo::needWorkchainInfo); cur_validators_.reset(); next_validators_.reset(); if (res.is_error()) { @@ -519,15 +520,15 @@ bool MasterchainStateQ::check_old_mc_block_id(const ton::BlockIdExt& blkid, bool return config_ && config_->check_old_mc_block_id(blkid, strict); } -td::uint32 MasterchainStateQ::min_split_depth(WorkchainId workchain_id) const { +td::uint32 MasterchainStateQ::monitor_min_split_depth(WorkchainId workchain_id) const { if (!config_) { return 0; } auto wc_info = config_->get_workchain_info(workchain_id); - return wc_info.not_null() ? wc_info->actual_min_split : 0; + return wc_info.not_null() ? wc_info->monitor_min_split : 0; } -td::uint32 MasterchainStateQ::soft_min_split_depth(WorkchainId workchain_id) const { +td::uint32 MasterchainStateQ::min_split_depth(WorkchainId workchain_id) const { if (!config_) { return 0; } @@ -564,5 +565,9 @@ BlockIdExt MasterchainStateQ::prev_key_block_id(BlockSeqno seqno) const { return block_id; } +bool MasterchainStateQ::is_key_state() const { + return config_ ? config_->is_key_state() : false; +} + } // namespace validator } // namespace ton diff --git a/validator/impl/shard.hpp b/validator/impl/shard.hpp index 99a9e8b08..174c5b622 100644 --- a/validator/impl/shard.hpp +++ b/validator/impl/shard.hpp @@ -120,8 +120,8 @@ class MasterchainStateQ : public MasterchainState, public ShardStateQ { bool has_workchain(WorkchainId workchain) const { return config_ && config_->has_workchain(workchain); } + td::uint32 monitor_min_split_depth(WorkchainId workchain_id) const override; td::uint32 min_split_depth(WorkchainId workchain_id) const override; - td::uint32 soft_min_split_depth(WorkchainId workchain_id) const override; BlockSeqno min_ref_masterchain_seqno() const override; td::Status prepare() override; ZeroStateIdExt get_zerostate_id() const { @@ -137,6 +137,7 @@ class MasterchainStateQ : public MasterchainState, public ShardStateQ { BlockIdExt last_key_block_id() const override; BlockIdExt next_key_block_id(BlockSeqno seqno) const override; BlockIdExt prev_key_block_id(BlockSeqno seqno) const override; + bool is_key_state() const override; MasterchainStateQ* make_copy() const override; static td::Result> fetch(const BlockIdExt& _id, td::BufferSlice _data, diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 003b7f9f7..8490567e8 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -57,7 +57,6 @@ std::string ErrorCtx::as_string() const { * Constructs a ValidateQuery object. * * @param shard The shard of the block being validated. - * @param min_ts The minimum allowed UnixTime for the block. * @param min_masterchain_block_id The minimum allowed masterchain block reference for the block. * @param prev A vector of BlockIdExt representing the previous blocks. * @param candidate The BlockCandidate to be validated. @@ -67,13 +66,12 @@ std::string ErrorCtx::as_string() const { * @param promise The Promise to return the ValidateCandidateResult to. * @param is_fake A boolean indicating if the validation is fake (performed when creating a hardfork). */ -ValidateQuery::ValidateQuery(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_masterchain_block_id, - std::vector prev, BlockCandidate candidate, Ref validator_set, +ValidateQuery::ValidateQuery(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, + BlockCandidate candidate, Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, bool is_fake) : shard_(shard) , id_(candidate.id) - , min_ts(min_ts) , min_mc_block_id(min_masterchain_block_id) , prev_blocks(std::move(prev)) , block_candidate(std::move(candidate)) @@ -87,7 +85,6 @@ ValidateQuery::ValidateQuery(ShardIdFull shard, UnixTime min_ts, BlockIdExt min_ , perf_timer_("validateblock", 0.1, [manager](double duration) { send_closure(manager, &ValidatorManager::add_perf_timer_stat, "validateblock", duration); }) { - proc_hash_.zero(); } /** @@ -709,7 +706,7 @@ void ValidateQuery::after_get_latest_mc_state(td::Result> res) { @@ -1333,7 +1330,7 @@ bool ValidateQuery::compute_next_state() { * Unpacks and merges the states of two previous blocks. * Used if the block is after_merge. * Similar to Collator::unpack_merge_last_state() - * + * * @returns True if the unpacking and merging was successful, false otherwise. */ bool ValidateQuery::unpack_merge_prev_state() { @@ -2334,7 +2331,7 @@ bool ValidateQuery::fix_all_processed_upto() { * Adds trivials neighbor after merging two shards. * Trivial neighbors are the two previous blocks. * Almost the same as in Collator. - * + * * @returns True if the operation is successful, false otherwise. */ bool ValidateQuery::add_trivial_neighbor_after_merge() { @@ -2810,7 +2807,7 @@ bool ValidateQuery::precheck_one_account_update(td::ConstBitPtr acc_id, Refsource).blk_.to_str()); } if (unprocessed) { - inbound_queues_empty_ = false; return true; } nb_out_msgs.next(); } - inbound_queues_empty_ = true; return true; } @@ -5777,7 +5772,7 @@ bool ValidateQuery::scan_account_libraries(Ref orig_libs, Ref prev, + ValidateQuery(ShardIdFull shard, BlockIdExt min_masterchain_block_id, std::vector prev, BlockCandidate candidate, td::Ref validator_set, td::actor::ActorId manager, td::Timestamp timeout, td::Promise promise, bool is_fake = false); @@ -127,7 +127,6 @@ class ValidateQuery : public td::actor::Actor { int pending{0}; const ShardIdFull shard_; const BlockIdExt id_; - UnixTime min_ts; BlockIdExt min_mc_block_id; std::vector prev_blocks; std::vector> prev_states; @@ -224,8 +223,7 @@ class ValidateQuery : public td::actor::Actor { td::RefInt256 import_fees_; ton::LogicalTime proc_lt_{0}, claimed_proc_lt_{0}, min_enq_lt_{~0ULL}; - ton::Bits256 proc_hash_, claimed_proc_hash_, min_enq_hash_; - bool inbound_queues_empty_{false}; + ton::Bits256 proc_hash_ = ton::Bits256::zero(), claimed_proc_hash_, min_enq_hash_; std::vector> msg_proc_lt_; std::vector> msg_emitted_lt_; diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index 8bbf7f31f..466203226 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -46,6 +46,7 @@ class Db : public td::actor::Actor { virtual void store_block_candidate(BlockCandidate candidate, td::Promise promise) = 0; virtual void get_block_candidate(ton::PublicKey source, BlockIdExt id, FileHash collated_data_file_hash, td::Promise promise) = 0; + virtual void get_block_candidate_by_block_id(BlockIdExt id, td::Promise promise) = 0; virtual void store_block_state(BlockHandle handle, td::Ref state, td::Promise> promise) = 0; diff --git a/validator/interfaces/shard.h b/validator/interfaces/shard.h index 35fe4bc9a..a31d178a5 100644 --- a/validator/interfaces/shard.h +++ b/validator/interfaces/shard.h @@ -70,14 +70,15 @@ class MasterchainState : virtual public ShardState { virtual std::vector> get_shards() const = 0; virtual td::Ref get_shard_from_config(ShardIdFull shard) const = 0; virtual bool workchain_is_active(WorkchainId workchain_id) const = 0; + virtual td::uint32 monitor_min_split_depth(WorkchainId workchain_id) const = 0; virtual td::uint32 min_split_depth(WorkchainId workchain_id) const = 0; - virtual td::uint32 soft_min_split_depth(WorkchainId workchain_id) const = 0; virtual BlockSeqno min_ref_masterchain_seqno() const = 0; virtual bool ancestor_is_valid(BlockIdExt id) const = 0; virtual ValidatorSessionConfig get_consensus_config() const = 0; virtual BlockIdExt last_key_block_id() const = 0; virtual BlockIdExt next_key_block_id(BlockSeqno seqno) const = 0; virtual BlockIdExt prev_key_block_id(BlockSeqno seqno) const = 0; + virtual bool is_key_state() const = 0; virtual bool get_old_mc_block_id(ton::BlockSeqno seqno, ton::BlockIdExt& blkid, ton::LogicalTime* end_lt = nullptr) const = 0; virtual bool check_old_mc_block_id(const ton::BlockIdExt& blkid, bool strict = false) const = 0; diff --git a/validator/manager-disk.cpp b/validator/manager-disk.cpp index 5678408c6..4c74ec60b 100644 --- a/validator/manager-disk.cpp +++ b/validator/manager-disk.cpp @@ -128,7 +128,7 @@ void ValidatorManagerImpl::sync_complete(td::Promise promise) { } Ed25519_PublicKey created_by{td::Bits256::zero()}; td::as(created_by.as_bits256().data() + 32 - 4) = ((unsigned)std::time(nullptr) >> 8); - run_collate_query(shard_id, 0, last_masterchain_block_id_, prev, created_by, val_set, td::Ref{true}, + run_collate_query(shard_id, last_masterchain_block_id_, prev, created_by, val_set, td::Ref{true}, actor_id(this), td::Timestamp::in(10.0), std::move(P)); } @@ -152,7 +152,7 @@ void ValidatorManagerImpl::validate_fake(BlockCandidate candidate, std::vector promise) { + td::actor::send_closure(db_, &Db::get_block_candidate_by_block_id, id, + promise.wrap([](BlockCandidate &&b) { return std::move(b.data); })); +} + void ValidatorManagerImpl::get_block_proof_from_db(ConstBlockHandle handle, td::Promise> promise) { td::actor::send_closure(db_, &Db::get_block_proof, std::move(handle), std::move(promise)); } diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index 3a77f2301..2f80e28ce 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -213,6 +213,7 @@ class ValidatorManagerImpl : public ValidatorManager { void get_shard_state_from_db_short(BlockIdExt block_id, td::Promise> promise) override; void get_block_candidate_from_db(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash, td::Promise promise) override; + void get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise promise) override; void get_block_proof_from_db(ConstBlockHandle handle, td::Promise> promise) override; void get_block_proof_from_db_short(BlockIdExt id, td::Promise> promise) override; void get_block_proof_link_from_db(ConstBlockHandle handle, td::Promise> promise) override; diff --git a/validator/manager-hardfork.cpp b/validator/manager-hardfork.cpp index 49d27085f..91c598aa2 100644 --- a/validator/manager-hardfork.cpp +++ b/validator/manager-hardfork.cpp @@ -415,6 +415,11 @@ void ValidatorManagerImpl::get_block_candidate_from_db(PublicKey source, BlockId td::actor::send_closure(db_, &Db::get_block_candidate, source, id, collated_data_file_hash, std::move(promise)); } +void ValidatorManagerImpl::get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise promise) { + td::actor::send_closure(db_, &Db::get_block_candidate_by_block_id, id, + promise.wrap([](BlockCandidate &&b) { return std::move(b.data); })); +} + void ValidatorManagerImpl::get_block_proof_from_db(ConstBlockHandle handle, td::Promise> promise) { td::actor::send_closure(db_, &Db::get_block_proof, std::move(handle), std::move(promise)); } diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index cf4d3799f..648a4a1b5 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -258,6 +258,7 @@ class ValidatorManagerImpl : public ValidatorManager { void get_shard_state_from_db_short(BlockIdExt block_id, td::Promise> promise) override; void get_block_candidate_from_db(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash, td::Promise promise) override; + void get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise promise) override; void get_block_proof_from_db(ConstBlockHandle handle, td::Promise> promise) override; void get_block_proof_from_db_short(BlockIdExt id, td::Promise> promise) override; void get_block_proof_link_from_db(ConstBlockHandle handle, td::Promise> promise) override; diff --git a/validator/manager.cpp b/validator/manager.cpp index fa592a788..79661a293 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -1035,6 +1035,16 @@ void ValidatorManagerImpl::get_block_candidate_from_db(PublicKey source, BlockId td::actor::send_closure(db_, &Db::get_block_candidate, source, id, collated_data_file_hash, std::move(promise)); } +void ValidatorManagerImpl::get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise promise) { + auto it = cached_block_candidates_.find(id); + if (it != cached_block_candidates_.end()) { + promise.set_result(it->second.data.clone()); + return; + } + td::actor::send_closure(db_, &Db::get_block_candidate_by_block_id, id, + promise.wrap([](BlockCandidate &&b) { return std::move(b.data); })); +} + void ValidatorManagerImpl::get_block_proof_from_db(ConstBlockHandle handle, td::Promise> promise) { td::actor::send_closure(db_, &Db::get_block_proof, std::move(handle), std::move(promise)); } @@ -2040,15 +2050,13 @@ void ValidatorManagerImpl::update_shards() { auto it2 = next_validator_groups_.find(legacy_val_group_id); if (it2 != next_validator_groups_.end()) { if (!it2->second.actor.empty()) { - td::actor::send_closure(it2->second.actor, &ValidatorGroup::start, prev, last_masterchain_block_id_, - last_masterchain_state_->get_unix_time()); + td::actor::send_closure(it2->second.actor, &ValidatorGroup::start, prev, last_masterchain_block_id_); } new_validator_groups_.emplace(val_group_id, std::move(it2->second)); } else { auto G = create_validator_group(val_group_id, shard, val_set, key_seqno, opts, started_); if (!G.empty()) { - td::actor::send_closure(G, &ValidatorGroup::start, prev, last_masterchain_block_id_, - last_masterchain_state_->get_unix_time()); + td::actor::send_closure(G, &ValidatorGroup::start, prev, last_masterchain_block_id_); } new_validator_groups_.emplace(val_group_id, ValidatorGroupEntry{std::move(G), shard}); } @@ -2096,15 +2104,13 @@ void ValidatorManagerImpl::update_shards() { auto it2 = next_validator_groups_.find(val_group_id); if (it2 != next_validator_groups_.end()) { if (!it2->second.actor.empty()) { - td::actor::send_closure(it2->second.actor, &ValidatorGroup::start, prev, last_masterchain_block_id_, - last_masterchain_state_->get_unix_time()); + td::actor::send_closure(it2->second.actor, &ValidatorGroup::start, prev, last_masterchain_block_id_); } new_validator_groups_.emplace(val_group_id, std::move(it2->second)); } else { auto G = create_validator_group(val_group_id, shard, val_set, key_seqno, opts, started_); if (!G.empty()) { - td::actor::send_closure(G, &ValidatorGroup::start, prev, last_masterchain_block_id_, - last_masterchain_state_->get_unix_time()); + td::actor::send_closure(G, &ValidatorGroup::start, prev, last_masterchain_block_id_); } new_validator_groups_.emplace(val_group_id, ValidatorGroupEntry{std::move(G), shard}); } diff --git a/validator/manager.hpp b/validator/manager.hpp index 99aa4e0e1..28cba9704 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -460,6 +460,7 @@ class ValidatorManagerImpl : public ValidatorManager { void get_shard_state_from_db_short(BlockIdExt block_id, td::Promise> promise) override; void get_block_candidate_from_db(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash, td::Promise promise) override; + void get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise promise) override; void get_block_proof_from_db(ConstBlockHandle handle, td::Promise> promise) override; void get_block_proof_from_db_short(BlockIdExt id, td::Promise> promise) override; void get_block_proof_link_from_db(ConstBlockHandle handle, td::Promise> promise) override; diff --git a/validator/net/download-block-new.cpp b/validator/net/download-block-new.cpp index 9ec36e335..e9a193b46 100644 --- a/validator/net/download-block-new.cpp +++ b/validator/net/download-block-new.cpp @@ -202,10 +202,10 @@ void DownloadBlockNew::got_node_to_download(adnl::AdnlNodeIdShort node) { } if (client_.empty()) { td::actor::send_closure(overlays_, &overlay::Overlays::send_query_via, download_from_, local_id_, overlay_id_, - "get_proof", std::move(P), td::Timestamp::in(15.0), std::move(q), + "get_block_full", std::move(P), td::Timestamp::in(15.0), std::move(q), FullNode::max_proof_size() + FullNode::max_block_size() + 128, rldp_); } else { - td::actor::send_closure(client_, &adnl::AdnlExtClient::send_query, "get_prepare", + td::actor::send_closure(client_, &adnl::AdnlExtClient::send_query, "get_block_full", create_serialize_tl_object_suffix(std::move(q)), td::Timestamp::in(15.0), std::move(P)); } diff --git a/validator/shard-client.cpp b/validator/shard-client.cpp index 8fad36122..62a762a5f 100644 --- a/validator/shard-client.cpp +++ b/validator/shard-client.cpp @@ -250,7 +250,7 @@ void ShardClient::build_shard_overlays() { for (auto &x : v) { auto shard = x->shard(); if (opts_->need_monitor(shard)) { - auto d = masterchain_state_->soft_min_split_depth(shard.workchain); + auto d = masterchain_state_->monitor_min_split_depth(shard.workchain); auto l = shard_prefix_length(shard.shard); if (l > d) { shard = shard_prefix(shard, d); diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 4b61c07cd..b3f327782 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -51,9 +51,9 @@ void ValidatorGroup::generate_block_candidate( return validatorsession::ValidatorSession::GeneratedCandidate{std::move(res), false}; })); run_collate_query( - shard_, min_ts_, min_masterchain_block_id_, prev_block_ids_, - Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, validator_set_, opts_->get_collator_options(), manager_, - td::Timestamp::in(10.0), [SelfId = actor_id(this), cache = cached_collated_block_](td::Result R) { + shard_, min_masterchain_block_id_, prev_block_ids_, Ed25519_PublicKey{local_id_full_.ed25519_value().raw()}, + validator_set_, opts_->get_collator_options(), manager_, td::Timestamp::in(10.0), + [SelfId = actor_id(this), cache = cached_collated_block_](td::Result R) { td::actor::send_closure(SelfId, &ValidatorGroup::generated_block_candidate, std::move(cache), std::move(R)); }); } @@ -132,8 +132,8 @@ void ValidatorGroup::validate_block_candidate(td::uint32 round_id, BlockCandidat } VLOG(VALIDATOR_DEBUG) << "validating block candidate " << next_block_id; block.id = next_block_id; - run_validate_query(shard_, min_ts_, min_masterchain_block_id_, prev_block_ids_, std::move(block), validator_set_, - manager_, td::Timestamp::in(15.0), std::move(P)); + run_validate_query(shard_, min_masterchain_block_id_, prev_block_ids_, std::move(block), validator_set_, manager_, + td::Timestamp::in(15.0), std::move(P)); } void ValidatorGroup::update_approve_cache(CacheKey key, UnixTime value) { @@ -357,10 +357,9 @@ void ValidatorGroup::create_session() { } } -void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterchain_block_id, UnixTime min_ts) { +void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterchain_block_id) { prev_block_ids_ = prev; min_masterchain_block_id_ = min_masterchain_block_id; - min_ts_ = min_ts; cached_collated_block_ = nullptr; approved_candidates_cache_.clear(); started_ = true; diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index 936d2fdc7..16e13b4b6 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -51,7 +51,7 @@ class ValidatorGroup : public td::actor::Actor { BlockIdExt create_next_block_id(RootHash root_hash, FileHash file_hash) const; BlockId create_next_block_id_simple() const; - void start(std::vector prev, BlockIdExt min_masterchain_block_id, UnixTime min_ts); + void start(std::vector prev, BlockIdExt min_masterchain_block_id); void create_session(); void destroy(); void start_up() override { @@ -114,7 +114,6 @@ class ValidatorGroup : public td::actor::Actor { std::vector prev_block_ids_; BlockIdExt min_masterchain_block_id_; - UnixTime min_ts_; td::Ref validator_set_; BlockSeqno last_key_block_seqno_; @@ -142,7 +141,7 @@ class ValidatorGroup : public td::actor::Actor { void generated_block_candidate(std::shared_ptr cache, td::Result R); - typedef std::tuple CacheKey; + using CacheKey = std::tuple; std::map approved_candidates_cache_; void update_approve_cache(CacheKey key, UnixTime value); diff --git a/validator/validator.h b/validator/validator.h index 3bceec6fe..46ee06340 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -247,6 +247,7 @@ class ValidatorManagerInterface : public td::actor::Actor { virtual void get_block_data_from_db_short(BlockIdExt block_id, td::Promise> promise) = 0; virtual void get_block_candidate_from_db(PublicKey source, BlockIdExt id, FileHash collated_data_file_hash, td::Promise promise) = 0; + virtual void get_candidate_data_by_block_id_from_db(BlockIdExt id, td::Promise promise) = 0; virtual void get_shard_state_from_db(ConstBlockHandle handle, td::Promise> promise) = 0; virtual void get_shard_state_from_db_short(BlockIdExt block_id, td::Promise> promise) = 0; virtual void get_block_proof_from_db(ConstBlockHandle handle, td::Promise> promise) = 0; From 1af2d3776f9c51d8e34fd5f63273437d72a183b1 Mon Sep 17 00:00:00 2001 From: neodix42 Date: Fri, 23 Aug 2024 11:59:40 +0300 Subject: [PATCH 09/37] * update links to docker image (#1109) * include fift and func into Docker image Co-authored-by: EmelyanenkoK Co-authored-by: neodiX --- Dockerfile | 6 +++++- docker/ton-ali.yaml | 2 +- docker/ton-aws.yaml | 2 +- docker/ton-gcp.yaml | 2 +- docker/ton-metal-lb.yaml | 2 +- docker/ton-node-port.yaml | 2 +- 6 files changed, 10 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index cf4187630..f4ea43759 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,7 +27,7 @@ RUN apt-get update && \ apt-get install -y wget curl libatomic1 openssl libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev libjemalloc-dev htop net-tools netcat iptraf-ng jq tcpdump pv plzip && \ rm -rf /var/lib/apt/lists/* -RUN mkdir -p /var/ton-work/db /var/ton-work/scripts +RUN mkdir -p /var/ton-work/db /var/ton-work/scripts /usr/share/ton/smartcont/ /usr/lib/fift/ COPY --from=builder /ton/build/storage/storage-daemon/storage-daemon /usr/local/bin/ COPY --from=builder /ton/build/storage/storage-daemon/storage-daemon-cli /usr/local/bin/ @@ -35,6 +35,10 @@ COPY --from=builder /ton/build/lite-client/lite-client /usr/local/bin/ COPY --from=builder /ton/build/validator-engine/validator-engine /usr/local/bin/ COPY --from=builder /ton/build/validator-engine-console/validator-engine-console /usr/local/bin/ COPY --from=builder /ton/build/utils/generate-random-id /usr/local/bin/ +COPY --from=builder /ton/build/crypto/fift /usr/local/bin/ +COPY --from=builder /ton/build/crypto/func /usr/local/bin/ +COPY --from=builder /ton/crypto/smartcont/* /usr/share/ton/smartcont/ +COPY --from=builder /ton/crypto/fift/lib/* /usr/lib/fift/ WORKDIR /var/ton-work/db COPY ./docker/init.sh ./docker/control.template /var/ton-work/scripts/ diff --git a/docker/ton-ali.yaml b/docker/ton-ali.yaml index 03ffbdb0f..2dd5daf6a 100644 --- a/docker/ton-ali.yaml +++ b/docker/ton-ali.yaml @@ -27,7 +27,7 @@ spec: spec: containers: - name: validator-engine-container - image: ghcr.io/neodix42/ton:latest + image: ghcr.io/ton-blockchain/ton:latest env: - name: PUBLIC_IP value: "" diff --git a/docker/ton-aws.yaml b/docker/ton-aws.yaml index da16cbae9..4cab1b556 100644 --- a/docker/ton-aws.yaml +++ b/docker/ton-aws.yaml @@ -27,7 +27,7 @@ spec: spec: containers: - name: validator-engine-container - image: ghcr.io/neodix42/ton:latest + image: ghcr.io/ton-blockchain/ton:latest env: - name: PUBLIC_IP value: "" diff --git a/docker/ton-gcp.yaml b/docker/ton-gcp.yaml index 0ded5a794..e3d89a06e 100644 --- a/docker/ton-gcp.yaml +++ b/docker/ton-gcp.yaml @@ -27,7 +27,7 @@ spec: spec: containers: - name: validator-engine-container - image: ghcr.io/neodix42/ton:latest + image: ghcr.io/ton-blockchain/ton:latest env: - name: PUBLIC_IP value: "" diff --git a/docker/ton-metal-lb.yaml b/docker/ton-metal-lb.yaml index ceaf3a7c0..1e62a45ab 100644 --- a/docker/ton-metal-lb.yaml +++ b/docker/ton-metal-lb.yaml @@ -11,7 +11,7 @@ spec: claimName: validator-engine-pvc containers: - name: validator-engine-container - image: ghcr.io/neodix42/ton:latest + image: ghcr.io/ton-blockchain/ton:latest env: - name: PUBLIC_IP value: "" diff --git a/docker/ton-node-port.yaml b/docker/ton-node-port.yaml index ec594031f..a9f1fe5c8 100644 --- a/docker/ton-node-port.yaml +++ b/docker/ton-node-port.yaml @@ -21,7 +21,7 @@ spec: claimName: validator-engine-pvc containers: - name: validator-engine-container - image: ghcr.io/neodix42/ton:latest + image: ghcr.io/ton-blockchain/ton:latest env: - name: PUBLIC_IP value: "" From cba92777a41b603b094b9c5f69d756a9255c350f Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 23 Aug 2024 13:12:40 +0300 Subject: [PATCH 10/37] Fix adding overlay neighbor (#1121) --- overlay/overlay-peers.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/overlay/overlay-peers.cpp b/overlay/overlay-peers.cpp index d37002444..3464671bb 100644 --- a/overlay/overlay-peers.cpp +++ b/overlay/overlay-peers.cpp @@ -213,9 +213,7 @@ void OverlayImpl::add_peer(OverlayNode node) { peer_list_.peers_.insert(id, OverlayPeer(std::move(node))); del_some_peers(); auto X = peer_list_.peers_.get(id); - CHECK(X); - - if (peer_list_.neighbours_.size() < max_neighbours() && + if (X != nullptr && peer_list_.neighbours_.size() < max_neighbours() && !(X->get_node()->flags() & OverlayMemberFlags::DoNotReceiveBroadcasts) && X->get_id() != local_id_) { peer_list_.neighbours_.push_back(X->get_id()); X->set_neighbour(true); From 16a2ced4d3c5f79e6deee34b7cdb9826da217db3 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 26 Aug 2024 17:53:42 +0300 Subject: [PATCH 11/37] Set default state ttl to 86400, set serializer delay to up to 6h (#1125) --- validator/state-serializer.cpp | 2 +- validator/validator.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index b27561b63..f7ea7efe9 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -160,7 +160,7 @@ void AsyncStateSerializer::next_iteration() { LOG(ERROR) << "started serializing persistent state for " << masterchain_handle_->id().id.to_str(); // block next attempts immediately, but send actual request later running_ = true; - double delay = td::Random::fast(0, 3600); + double delay = td::Random::fast(0, 3600 * 6); LOG(WARNING) << "serializer delay = " << delay << "s"; delay_action( [SelfId = actor_id(this)]() { diff --git a/validator/validator.h b/validator/validator.h index 46ee06340..f326c9012 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -145,7 +145,7 @@ struct ValidatorManagerOptions : public td::CntObject { std::function check_shard = [](ShardIdFull, CatchainSeqno, ShardCheckMode) { return true; }, bool allow_blockchain_init = false, double sync_blocks_before = 3600, double block_ttl = 86400, - double state_ttl = 3600, double archive_ttl = 86400 * 7, double key_proof_ttl = 86400 * 3650, + double state_ttl = 86400, double archive_ttl = 86400 * 7, double key_proof_ttl = 86400 * 3650, double max_mempool_num = 999999, bool initial_sync_disabled = false); }; From 6038147afe999be5eadae332593cf97a0ce3ee47 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 27 Aug 2024 18:10:17 +0300 Subject: [PATCH 12/37] Disable state serializer on masterchain validators (#1129) --- validator/manager.cpp | 12 ++++++ validator/state-serializer.cpp | 72 ++++++++++++++++++++-------------- validator/state-serializer.hpp | 13 ++++-- 3 files changed, 64 insertions(+), 33 deletions(-) diff --git a/validator/manager.cpp b/validator/manager.cpp index 79661a293..f127b1fd4 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2065,6 +2065,7 @@ void ValidatorManagerImpl::update_shards() { } } + bool validating_masterchain = false; if (allow_validate_) { for (auto &desc : new_shards) { auto shard = desc.first; @@ -2081,6 +2082,9 @@ void ValidatorManagerImpl::update_shards() { auto validator_id = get_validator(shard, val_set); if (!validator_id.is_zero()) { + if (shard.is_masterchain()) { + validating_masterchain = true; + } auto val_group_id = get_validator_set_id(shard, val_set, opts_hash, key_seqno, opts); if (force_recover) { @@ -2173,6 +2177,14 @@ void ValidatorManagerImpl::update_shards() { td::actor::send_closure(SelfId, &ValidatorManagerImpl::written_destroyed_validator_sessions, std::move(gc)); }); td::actor::send_closure(db_, &Db::update_destroyed_validator_sessions, gc_list_, std::move(P)); + + if (!serializer_.empty()) { + td::actor::send_closure( + serializer_, &AsyncStateSerializer::auto_disable_serializer, + validating_masterchain && + last_masterchain_state_->get_validator_set(ShardIdFull{masterchainId})->export_vector().size() * 2 <= + last_masterchain_state_->get_total_validator_set(0)->export_vector().size()); + } } } // namespace validator diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index f7ea7efe9..ef79d33cb 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -95,7 +95,8 @@ void AsyncStateSerializer::request_previous_state_files() { } void AsyncStateSerializer::got_previous_state_files(std::vector> files) { - previous_state_files_ = std::move(files); + previous_state_cache_ = std::make_shared(); + previous_state_cache_->state_files = std::move(files); request_masterchain_state(); } @@ -151,7 +152,10 @@ void AsyncStateSerializer::next_iteration() { need_serialize(masterchain_handle_)) { if (!have_masterchain_state_ && !opts_->get_state_serializer_enabled()) { LOG(ERROR) << "skipping serializing persistent state for " << masterchain_handle_->id().id.to_str() - << ": serializer is disabled"; + << ": serializer is disabled (by user)"; + } else if (!have_masterchain_state_ && auto_disabled_) { + LOG(ERROR) << "skipping serializing persistent state for " << masterchain_handle_->id().id.to_str() + << ": serializer is disabled (automatically)"; } else if (!have_masterchain_state_ && have_newer_persistent_state(masterchain_handle_->unix_time())) { LOG(ERROR) << "skipping serializing persistent state for " << masterchain_handle_->id().id.to_str() << ": newer key block with ts=" << last_known_key_block_ts_ << " exists"; @@ -182,9 +186,7 @@ void AsyncStateSerializer::next_iteration() { } last_key_block_ts_ = masterchain_handle_->unix_time(); last_key_block_id_ = masterchain_handle_->id(); - previous_state_files_ = {}; previous_state_cache_ = {}; - previous_state_cur_shards_ = {}; } if (!saved_to_db_) { running_ = true; @@ -252,27 +254,24 @@ class CachedCellDbReader : public vm::CellDbReader { td::uint64 cached_reqs_ = 0; }; -void AsyncStateSerializer::prepare_previous_state_cache(ShardIdFull shard) { - if (!opts_->get_fast_state_serializer_enabled()) { - return; - } +void AsyncStateSerializer::PreviousStateCache::prepare_cache(ShardIdFull shard) { std::vector prev_shards; - for (const auto& [_, prev_shard] : previous_state_files_) { + for (const auto& [_, prev_shard] : state_files) { if (shard_intersects(shard, prev_shard)) { prev_shards.push_back(prev_shard); } } - if (prev_shards == previous_state_cur_shards_) { + if (prev_shards == cur_shards) { return; } - previous_state_cur_shards_ = std::move(prev_shards); - previous_state_cache_ = {}; - if (previous_state_cur_shards_.empty()) { + cur_shards = std::move(prev_shards); + cache = {}; + if (cur_shards.empty()) { return; } td::Timer timer; LOG(WARNING) << "Preloading previous persistent state for shard " << shard.to_str() << " (" - << previous_state_cur_shards_.size() << " files)"; + << cur_shards.size() << " files)"; std::map> cells; std::function)> dfs = [&](td::Ref cell) { td::Bits256 hash = cell->get_hash().bits(); @@ -285,7 +284,7 @@ void AsyncStateSerializer::prepare_previous_state_cache(ShardIdFull shard) { dfs(cs.prefetch_ref(i)); } }; - for (const auto& [file, prev_shard] : previous_state_files_) { + for (const auto& [file, prev_shard] : state_files) { if (!shard_intersects(shard, prev_shard)) { continue; } @@ -300,22 +299,20 @@ void AsyncStateSerializer::prepare_previous_state_cache(ShardIdFull shard) { LOG(WARNING) << "Deserialize error : " << r_root.move_as_error(); continue; } - r_data = {}; + r_data.clear(); dfs(r_root.move_as_ok()); } LOG(WARNING) << "Preloaded previous state: " << cells.size() << " cells in " << timer.elapsed() << "s"; - previous_state_cache_ = std::make_shared>>(std::move(cells)); + cache = std::make_shared>>(std::move(cells)); } void AsyncStateSerializer::got_masterchain_state(td::Ref state, std::shared_ptr cell_db_reader) { - if (!opts_->get_state_serializer_enabled()) { + if (!opts_->get_state_serializer_enabled() || auto_disabled_) { stored_masterchain_state(); return; } LOG(ERROR) << "serializing masterchain state " << masterchain_handle_->id().id.to_str(); - prepare_previous_state_cache(state->get_shard()); - auto new_cell_db_reader = std::make_shared(cell_db_reader, previous_state_cache_); have_masterchain_state_ = true; CHECK(next_idx_ == 0); CHECK(shards_.size() == 0); @@ -325,10 +322,16 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref state shards_.push_back(v->top_block_id()); } - auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader = new_cell_db_reader, + auto write_data = [shard = state->get_shard(), hash = state->root_cell()->get_hash(), cell_db_reader, + previous_state_cache = previous_state_cache_, + fast_serializer_enabled = opts_->get_fast_state_serializer_enabled(), cancellation_token = cancellation_token_source_.get_cancellation_token()](td::FileFd& fd) mutable { - auto res = vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31, std::move(cancellation_token)); - cell_db_reader->print_stats(); + if (fast_serializer_enabled) { + previous_state_cache->prepare_cache(shard); + } + auto new_cell_db_reader = std::make_shared(cell_db_reader, previous_state_cache->cache); + auto res = vm::std_boc_serialize_to_file_large(new_cell_db_reader, hash, fd, 31, std::move(cancellation_token)); + new_cell_db_reader->print_stats(); return res; }; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { @@ -375,17 +378,21 @@ void AsyncStateSerializer::got_shard_handle(BlockHandle handle) { void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Ref state, std::shared_ptr cell_db_reader) { - if (!opts_->get_state_serializer_enabled()) { + if (!opts_->get_state_serializer_enabled() || auto_disabled_) { success_handler(); return; } LOG(ERROR) << "serializing shard state " << handle->id().id.to_str(); - prepare_previous_state_cache(state->get_shard()); - auto new_cell_db_reader = std::make_shared(cell_db_reader, previous_state_cache_); - auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader = new_cell_db_reader, + auto write_data = [shard = state->get_shard(), hash = state->root_cell()->get_hash(), cell_db_reader, + previous_state_cache = previous_state_cache_, + fast_serializer_enabled = opts_->get_fast_state_serializer_enabled(), cancellation_token = cancellation_token_source_.get_cancellation_token()](td::FileFd& fd) mutable { - auto res = vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31, std::move(cancellation_token)); - cell_db_reader->print_stats(); + if (fast_serializer_enabled) { + previous_state_cache->prepare_cache(shard); + } + auto new_cell_db_reader = std::make_shared(cell_db_reader, previous_state_cache->cache); + auto res = vm::std_boc_serialize_to_file_large(new_cell_db_reader, hash, fd, 31, std::move(cancellation_token)); + new_cell_db_reader->print_stats(); return res; }; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result R) { @@ -427,6 +434,13 @@ void AsyncStateSerializer::update_options(td::Ref opts) } } +void AsyncStateSerializer::auto_disable_serializer(bool disabled) { + auto_disabled_ = disabled; + if (auto_disabled_) { + cancellation_token_source_.cancel(); + } +} + bool AsyncStateSerializer::need_monitor(ShardIdFull shard) { return opts_->need_monitor(shard); diff --git a/validator/state-serializer.hpp b/validator/state-serializer.hpp index 6d966f930..68606d1ea 100644 --- a/validator/state-serializer.hpp +++ b/validator/state-serializer.hpp @@ -37,6 +37,7 @@ class AsyncStateSerializer : public td::actor::Actor { bool saved_to_db_ = true; td::Ref opts_; + bool auto_disabled_ = false; td::CancellationTokenSource cancellation_token_source_; UnixTime last_known_key_block_ts_ = 0; @@ -48,11 +49,14 @@ class AsyncStateSerializer : public td::actor::Actor { bool have_masterchain_state_ = false; std::vector shards_; - std::vector> previous_state_files_; - std::shared_ptr>> previous_state_cache_; - std::vector previous_state_cur_shards_; + struct PreviousStateCache { + std::vector> state_files; + std::shared_ptr>> cache; + std::vector cur_shards; - void prepare_previous_state_cache(ShardIdFull shard); + void prepare_cache(ShardIdFull shard); + }; + std::shared_ptr previous_state_cache_; public: AsyncStateSerializer(BlockIdExt block_id, td::Ref opts, @@ -105,6 +109,7 @@ class AsyncStateSerializer : public td::actor::Actor { void success_handler(); void update_options(td::Ref opts); + void auto_disable_serializer(bool disabled); }; } // namespace validator From be55da5fdef7340ee4e7f91015d53e38e4d6fbb0 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Tue, 27 Aug 2024 18:17:43 +0300 Subject: [PATCH 13/37] Increase moderate misbehavior threshold --- lite-client/lite-client.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index 02a5fab67..da2fd6ff9 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -3737,10 +3737,10 @@ void TestNode::continue_check_validator_load3(std::unique_ptr Date: Thu, 15 Aug 2024 12:53:05 +0300 Subject: [PATCH 14/37] Fix UB in blst aggregate verify (#1107) --- crypto/vm/bls.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/crypto/vm/bls.cpp b/crypto/vm/bls.cpp index f6ccc275c..ff5179c72 100644 --- a/crypto/vm/bls.cpp +++ b/crypto/vm/bls.cpp @@ -93,12 +93,13 @@ bool aggregate_verify(const std::vector> &pubs_ms return false; } std::unique_ptr pairing = std::make_unique(true, DST); + blst::P2_Affine p2_zero; for (const auto &p : pubs_msgs) { blst::P1_Affine p1(p.first.data(), P1_SIZE); if (!p1.in_group() || p1.is_inf()) { return false; } - pairing->aggregate(&p1, nullptr, (const td::uint8 *)p.second.data(), p.second.size()); + pairing->aggregate(&p1, &p2_zero, (const td::uint8 *)p.second.data(), p.second.size()); } pairing->commit(); blst::P2_Affine p2(sig.data(), P2_SIZE); From 6515136061b465f41a9a9f4c548d3da09ec0a154 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 15 Aug 2024 15:25:16 +0300 Subject: [PATCH 15/37] Improve creating channels in adnl (#1108) * Improve creating channels in adnl * Improve handling of cryptographic keys --- adnl/adnl-peer.cpp | 22 +++++++++++++++------ adnl/adnl-peer.hpp | 3 ++- keyring/keyring.cpp | 38 ++++++++++++++++++++---------------- keyring/keyring.hpp | 7 +++---- keys/encryptor.cpp | 22 --------------------- keys/encryptor.h | 22 --------------------- keys/encryptor.hpp | 14 ++++++++++---- keys/keys.cpp | 47 +++++++++++++++++++++++++++++++++++++++++---- keys/keys.hpp | 14 ++++++++++++++ 9 files changed, 109 insertions(+), 80 deletions(-) diff --git a/adnl/adnl-peer.cpp b/adnl/adnl-peer.cpp index febbdac6e..d82486fed 100644 --- a/adnl/adnl-peer.cpp +++ b/adnl/adnl-peer.cpp @@ -269,7 +269,11 @@ void AdnlPeerPairImpl::send_messages_in(std::vector message size_t ptr = 0; bool first = true; do { + respond_with_nop_after_ = td::Timestamp::in(td::Random::fast(1.0, 2.0)); bool try_reinit = try_reinit_at_ && try_reinit_at_.is_in_past(); + if (try_reinit) { + try_reinit_at_ = td::Timestamp::in(td::Random::fast(0.5, 1.5)); + } bool via_channel = channel_ready_ && !try_reinit; size_t s = (via_channel ? channel_packet_header_max_size() : packet_header_max_size()); if (first) { @@ -504,12 +508,6 @@ void AdnlPeerPairImpl::create_channel(pubkeys::Ed25519 pub, td::uint32 date) { void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageCreateChannel &message) { create_channel(message.key(), message.date()); - if (respond_to_channel_create_after_.is_in_past()) { - respond_to_channel_create_after_ = td::Timestamp::in(td::Random::fast(1.0, 2.0)); - std::vector messages; - messages.emplace_back(adnlmessage::AdnlMessageNop{}, 0); - send_messages(std::move(messages)); - } } void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageConfirmChannel &message) { @@ -526,6 +524,7 @@ void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageConfirmChan } void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageCustom &message) { + respond_with_nop(); td::actor::send_closure(local_actor_, &AdnlLocalId::deliver, peer_id_short_, message.data()); } @@ -538,6 +537,7 @@ void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageReinit &mes } void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageQuery &message) { + respond_with_nop(); auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), query_id = message.query_id(), flags = static_cast(0)](td::Result R) { if (R.is_error()) { @@ -556,6 +556,7 @@ void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageQuery &mess } void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageAnswer &message) { + respond_with_nop(); auto Q = out_queries_.find(message.query_id()); if (Q == out_queries_.end()) { @@ -573,6 +574,7 @@ void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageAnswer &mes } void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessagePart &message) { + respond_with_nop(); auto size = message.total_size(); if (size > huge_packet_max_size()) { VLOG(ADNL_WARNING) << this << ": dropping too big huge message: size=" << size; @@ -635,6 +637,14 @@ void AdnlPeerPairImpl::delete_query(AdnlQueryId id) { } } +void AdnlPeerPairImpl::respond_with_nop() { + if (respond_with_nop_after_.is_in_past()) { + std::vector messages; + messages.emplace_back(adnlmessage::AdnlMessageNop{}, 0); + send_messages(std::move(messages)); + } +} + void AdnlPeerPairImpl::reinit(td::int32 date) { if (reinit_date_ == 0) { reinit_date_ = date; diff --git a/adnl/adnl-peer.hpp b/adnl/adnl-peer.hpp index e9a5d428e..40c9eb088 100644 --- a/adnl/adnl-peer.hpp +++ b/adnl/adnl-peer.hpp @@ -122,6 +122,7 @@ class AdnlPeerPairImpl : public AdnlPeerPair { } private: + void respond_with_nop(); void reinit(td::int32 date); td::Result, bool>> get_conn(bool direct_only); void create_channel(pubkeys::Ed25519 pub, td::uint32 date); @@ -214,7 +215,7 @@ class AdnlPeerPairImpl : public AdnlPeerPair { pubkeys::Ed25519 channel_pub_; td::int32 channel_pk_date_; td::actor::ActorOwn channel_; - td::Timestamp respond_to_channel_create_after_; + td::Timestamp respond_with_nop_after_; td::uint64 in_seqno_ = 0; td::uint64 out_seqno_ = 0; diff --git a/keyring/keyring.cpp b/keyring/keyring.cpp index 529a6b8b0..0f45879de 100644 --- a/keyring/keyring.cpp +++ b/keyring/keyring.cpp @@ -27,6 +27,16 @@ namespace ton { namespace keyring { +KeyringImpl::PrivateKeyDescr::PrivateKeyDescr(PrivateKey private_key, bool is_temp) + : public_key(private_key.compute_public_key()), is_temp(is_temp) { + auto D = private_key.create_decryptor_async(); + D.ensure(); + decryptor_sign = D.move_as_ok(); + D = private_key.create_decryptor_async(); + D.ensure(); + decryptor_decrypt = D.move_as_ok(); +} + void KeyringImpl::start_up() { if (db_root_.size() > 0) { td::mkdir(db_root_).ensure(); @@ -45,23 +55,19 @@ td::Result KeyringImpl::load_key(PublicKeyHash k auto name = db_root_ + "/" + key_hash.bits256_value().to_hex(); - auto R = td::read_file(td::CSlice{name}); + auto R = td::read_file_secure(td::CSlice{name}); if (R.is_error()) { return R.move_as_error_prefix("key not in db: "); } auto data = R.move_as_ok(); - auto R2 = PrivateKey::import(td::SecureString(data)); + auto R2 = PrivateKey::import(data); R2.ensure(); auto key = R2.move_as_ok(); - auto pub = key.compute_public_key(); - auto short_id = pub.compute_short_id(); + auto desc = std::make_unique(key, false); + auto short_id = desc->public_key.compute_short_id(); CHECK(short_id == key_hash); - - auto D = key.create_decryptor_async(); - D.ensure(); - - return map_.emplace(short_id, std::make_unique(D.move_as_ok(), pub, false)).first->second.get(); + return map_.emplace(short_id, std::move(desc)).first->second.get(); } void KeyringImpl::add_key(PrivateKey key, bool is_temp, td::Promise promise) { @@ -76,10 +82,7 @@ void KeyringImpl::add_key(PrivateKey key, bool is_temp, td::Promise pr if (db_root_.size() == 0) { CHECK(is_temp); } - auto D = key.create_decryptor_async(); - D.ensure(); - - map_.emplace(short_id, std::make_unique(D.move_as_ok(), pub, is_temp)); + map_.emplace(short_id, std::make_unique(key, is_temp)); if (!is_temp && key.exportable()) { auto S = key.export_as_slice(); @@ -139,7 +142,7 @@ void KeyringImpl::sign_message(PublicKeyHash key_hash, td::BufferSlice data, td: if (S.is_error()) { promise.set_error(S.move_as_error()); } else { - td::actor::send_closure(S.move_as_ok()->decryptor, &DecryptorAsync::sign, std::move(data), std::move(promise)); + td::actor::send_closure(S.move_as_ok()->decryptor_sign, &DecryptorAsync::sign, std::move(data), std::move(promise)); } } @@ -161,7 +164,7 @@ void KeyringImpl::sign_add_get_public_key(PublicKeyHash key_hash, td::BufferSlic } promise.set_value(std::pair{R.move_as_ok(), id}); }); - td::actor::send_closure(D->decryptor, &DecryptorAsync::sign, std::move(data), std::move(P)); + td::actor::send_closure(D->decryptor_sign, &DecryptorAsync::sign, std::move(data), std::move(P)); } void KeyringImpl::sign_messages(PublicKeyHash key_hash, std::vector data, @@ -171,7 +174,7 @@ void KeyringImpl::sign_messages(PublicKeyHash key_hash, std::vectordecryptor, &DecryptorAsync::sign_batch, std::move(data), + td::actor::send_closure(S.move_as_ok()->decryptor_sign, &DecryptorAsync::sign_batch, std::move(data), std::move(promise)); } } @@ -182,7 +185,8 @@ void KeyringImpl::decrypt_message(PublicKeyHash key_hash, td::BufferSlice data, if (S.is_error()) { promise.set_error(S.move_as_error()); } else { - td::actor::send_closure(S.move_as_ok()->decryptor, &DecryptorAsync::decrypt, std::move(data), std::move(promise)); + td::actor::send_closure(S.move_as_ok()->decryptor_decrypt, &DecryptorAsync::decrypt, std::move(data), + std::move(promise)); } } diff --git a/keyring/keyring.hpp b/keyring/keyring.hpp index fc67bd0fe..ec658305a 100644 --- a/keyring/keyring.hpp +++ b/keyring/keyring.hpp @@ -30,12 +30,11 @@ namespace keyring { class KeyringImpl : public Keyring { private: struct PrivateKeyDescr { - td::actor::ActorOwn decryptor; + td::actor::ActorOwn decryptor_sign; + td::actor::ActorOwn decryptor_decrypt; PublicKey public_key; bool is_temp; - PrivateKeyDescr(td::actor::ActorOwn decryptor, PublicKey public_key, bool is_temp) - : decryptor(std::move(decryptor)), public_key(public_key), is_temp(is_temp) { - } + PrivateKeyDescr(PrivateKey private_key, bool is_temp); }; public: diff --git a/keys/encryptor.cpp b/keys/encryptor.cpp index 0b93d36fc..8fef9a095 100644 --- a/keys/encryptor.cpp +++ b/keys/encryptor.cpp @@ -29,28 +29,6 @@ namespace ton { -td::Result> Encryptor::create(const ton_api::PublicKey *id) { - td::Result> res; - ton_api::downcast_call( - *const_cast(id), - td::overloaded([&](const ton_api::pub_unenc &obj) { res = std::make_unique(); }, - [&](const ton_api::pub_ed25519 &obj) { res = std::make_unique(obj.key_); }, - [&](const ton_api::pub_overlay &obj) { res = std::make_unique(); }, - [&](const ton_api::pub_aes &obj) { res = std::make_unique(obj.key_); })); - return res; -} - -td::Result> Decryptor::create(const ton_api::PrivateKey *id) { - td::Result> res; - ton_api::downcast_call( - *const_cast(id), - td::overloaded([&](const ton_api::pk_unenc &obj) { res = std::make_unique(); }, - [&](const ton_api::pk_ed25519 &obj) { res = std::make_unique(obj.key_); }, - [&](const ton_api::pk_overlay &obj) { res = std::make_unique(); }, - [&](const ton_api::pk_aes &obj) { res = std::make_unique(obj.key_); })); - return res; -} - td::Result EncryptorEd25519::encrypt(td::Slice data) { TRY_RESULT_PREFIX(pk, td::Ed25519::generate_private_key(), "failed to generate private key: "); TRY_RESULT_PREFIX(pubkey, pk.get_public_key(), "failed to get public key from private: "); diff --git a/keys/encryptor.h b/keys/encryptor.h index 3035a0cec..818c97d63 100644 --- a/keys/encryptor.h +++ b/keys/encryptor.h @@ -31,7 +31,6 @@ class Encryptor { virtual td::Result encrypt(td::Slice data) = 0; virtual td::Status check_signature(td::Slice message, td::Slice signature) = 0; virtual ~Encryptor() = default; - static td::Result> create(const ton_api::PublicKey *id); }; class Decryptor { @@ -40,7 +39,6 @@ class Decryptor { virtual td::Result sign(td::Slice data) = 0; virtual std::vector> sign_batch(std::vector data); virtual ~Decryptor() = default; - static td::Result> create(const ton_api::PrivateKey *id); }; class EncryptorAsync : public td::actor::Actor { @@ -61,16 +59,6 @@ class EncryptorAsync : public td::actor::Actor { void encrypt(td::BufferSlice data, td::Promise promise) { promise.set_result(encryptor_->encrypt(data.as_slice())); } - template - static td::Result> create(T &id) { - TRY_RESULT(d, Encryptor::create(id)); - return td::actor::create_actor("encryptor", std::move(d)); - } - template - static td::Result> create(T *id) { - TRY_RESULT(d, Encryptor::create(id)); - return td::actor::create_actor("encryptor", std::move(d)); - } }; class DecryptorAsync : public td::actor::Actor { @@ -94,16 +82,6 @@ class DecryptorAsync : public td::actor::Actor { } return decryptor_->sign_batch(v); } - template - static td::Result> create(T &id) { - TRY_RESULT(d, Decryptor::create(id)); - return td::actor::create_actor("decryptor", std::move(d)); - } - template - static td::Result> create(T *id) { - TRY_RESULT(d, Decryptor::create(id)); - return td::actor::create_actor("decryptor", std::move(d)); - } }; } // namespace ton diff --git a/keys/encryptor.hpp b/keys/encryptor.hpp index dbe882398..bcc841dc8 100644 --- a/keys/encryptor.hpp +++ b/keys/encryptor.hpp @@ -83,7 +83,7 @@ class EncryptorEd25519 : public Encryptor { td::Result encrypt(td::Slice data) override; td::Status check_signature(td::Slice message, td::Slice signature) override; - EncryptorEd25519(td::Bits256 key) : pub_(td::SecureString(as_slice(key))) { + EncryptorEd25519(const td::Bits256& key) : pub_(td::SecureString(as_slice(key))) { } }; @@ -94,7 +94,7 @@ class DecryptorEd25519 : public Decryptor { public: td::Result decrypt(td::Slice data) override; td::Result sign(td::Slice data) override; - DecryptorEd25519(td::Bits256 key) : pk_(td::SecureString(as_slice(key))) { + DecryptorEd25519(const td::Bits256& key) : pk_(td::SecureString(as_slice(key))) { } }; @@ -129,12 +129,15 @@ class EncryptorAES : public Encryptor { td::Bits256 shared_secret_; public: + ~EncryptorAES() override { + shared_secret_.set_zero_s(); + } td::Result encrypt(td::Slice data) override; td::Status check_signature(td::Slice message, td::Slice signature) override { return td::Status::Error("can no sign channel messages"); } - EncryptorAES(td::Bits256 shared_secret) : shared_secret_(shared_secret) { + EncryptorAES(const td::Bits256& shared_secret) : shared_secret_(shared_secret) { } }; @@ -143,11 +146,14 @@ class DecryptorAES : public Decryptor { td::Bits256 shared_secret_; public: + ~DecryptorAES() override { + shared_secret_.set_zero_s(); + } td::Result decrypt(td::Slice data) override; td::Result sign(td::Slice data) override { return td::Status::Error("can no sign channel messages"); } - DecryptorAES(td::Bits256 shared_secret) : shared_secret_(shared_secret) { + DecryptorAES(const td::Bits256& shared_secret) : shared_secret_(shared_secret) { } }; diff --git a/keys/keys.cpp b/keys/keys.cpp index 7d6c0c2c4..01afb26d2 100644 --- a/keys/keys.cpp +++ b/keys/keys.cpp @@ -21,6 +21,7 @@ #include "td/utils/overloaded.h" #include "tl-utils/tl-utils.hpp" #include "encryptor.h" +#include "encryptor.hpp" #include "crypto/Ed25519.h" namespace ton { @@ -63,12 +64,31 @@ td::Result PublicKey::import(td::Slice s) { return PublicKey{x}; } +td::Result> pubkeys::Ed25519::create_encryptor() const { + return std::make_unique(data_); +} + +td::Result> pubkeys::AES::create_encryptor() const { + return std::make_unique(data_); +} + +td::Result> pubkeys::Unenc::create_encryptor() const { + return std::make_unique(); +} + +td::Result> pubkeys::Overlay::create_encryptor() const { + return std::make_unique(); +} + td::Result> PublicKey::create_encryptor() const { - return Encryptor::create(tl().get()); + td::Result> res; + pub_key_.visit([&](auto &obj) { res = obj.create_encryptor(); }); + return res; } td::Result> PublicKey::create_encryptor_async() const { - return EncryptorAsync::create(tl().get()); + TRY_RESULT(encryptor, create_encryptor()); + return td::actor::create_actor("encryptor", std::move(encryptor)); } bool PublicKey::empty() const { @@ -109,6 +129,22 @@ privkeys::Ed25519::Ed25519(td::Ed25519::PrivateKey key) { data_.as_slice().copy_from(td::Slice(s)); } +td::Result> privkeys::Ed25519::create_decryptor() const { + return std::make_unique(data_); +} + +td::Result> privkeys::AES::create_decryptor() const { + return std::make_unique(data_); +} + +td::Result> privkeys::Unenc::create_decryptor() const { + return std::make_unique(); +} + +td::Result> privkeys::Overlay::create_decryptor() const { + return std::make_unique(); +} + pubkeys::Ed25519::Ed25519(td::Ed25519::PublicKey key) { auto s = key.as_octet_string(); CHECK(s.length() == 32); @@ -188,11 +224,14 @@ tl_object_ptr PrivateKey::tl() const { } td::Result> PrivateKey::create_decryptor() const { - return Decryptor::create(tl().get()); + td::Result> res; + priv_key_.visit([&](auto &obj) { res = obj.create_decryptor(); }); + return res; } td::Result> PrivateKey::create_decryptor_async() const { - return DecryptorAsync::create(tl().get()); + TRY_RESULT(decryptor, create_decryptor()); + return td::actor::create_actor("decryptor", std::move(decryptor)); } } // namespace ton diff --git a/keys/keys.hpp b/keys/keys.hpp index cf883bbe6..72d0845ac 100644 --- a/keys/keys.hpp +++ b/keys/keys.hpp @@ -110,6 +110,7 @@ class Ed25519 { tl_object_ptr tl() const { return create_tl_object(data_); } + td::Result> create_encryptor() const; bool operator==(const Ed25519 &with) const { return data_ == with.data_; } @@ -141,6 +142,7 @@ class AES { tl_object_ptr tl() const { return create_tl_object(data_); } + td::Result> create_encryptor() const; bool operator==(const AES &with) const { return data_ == with.data_; } @@ -172,6 +174,7 @@ class Unenc { tl_object_ptr tl() const { return create_tl_object(data_.clone_as_buffer_slice()); } + td::Result> create_encryptor() const; bool operator==(const Unenc &with) const { return data_.as_slice() == with.data_.as_slice(); } @@ -203,6 +206,7 @@ class Overlay { tl_object_ptr tl() const { return create_tl_object(data_.clone_as_buffer_slice()); } + td::Result> create_encryptor() const; bool operator==(const Overlay &with) const { return data_.as_slice() == with.data_.as_slice(); } @@ -223,6 +227,9 @@ class PublicKey { td::uint32 serialized_size() const { UNREACHABLE(); } + td::Result> create_encryptor() const { + UNREACHABLE(); + } bool operator==(const Empty &with) const { return false; } @@ -316,6 +323,7 @@ class Ed25519 { } tl_object_ptr pub_tl() const; pubkeys::Ed25519 pub() const; + td::Result> create_decryptor() const; static Ed25519 random(); }; @@ -359,6 +367,7 @@ class AES { pubkeys::AES pub() const { return pubkeys::AES{data_}; } + td::Result> create_decryptor() const; }; class Unenc { @@ -393,6 +402,7 @@ class Unenc { pubkeys::Unenc pub() const { return pubkeys::Unenc{data_.clone()}; } + td::Result> create_decryptor() const; }; class Overlay { @@ -427,6 +437,7 @@ class Overlay { pubkeys::Overlay pub() const { return pubkeys::Overlay{data_.clone()}; } + td::Result> create_decryptor() const; }; } // namespace privkeys @@ -450,6 +461,9 @@ class PrivateKey { PublicKey pub() const { UNREACHABLE(); } + td::Result> create_decryptor() const { + UNREACHABLE(); + } }; td::Variant priv_key_{Empty{}}; From a71d4132022fb4743a275de4ddefbd1fe9e649a1 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 15 Aug 2024 15:26:35 +0300 Subject: [PATCH 16/37] Improve dht lookup in overlays (#1104) Continue dht lookup even if value was found --- dht/dht-in.hpp | 1 + dht/dht-query.cpp | 34 +++++++++++++++++++++++--- dht/dht-query.hpp | 51 +++++++++++++++++++++++++++++++++------ dht/dht.cpp | 17 ++++++++++++- dht/dht.h | 1 + overlay/overlay.cpp | 49 +++++++++++++++++++------------------ overlay/overlay.hpp | 3 ++- tdutils/td/utils/Status.h | 6 +++++ 8 files changed, 127 insertions(+), 35 deletions(-) diff --git a/dht/dht-in.hpp b/dht/dht-in.hpp index c2d20455f..0d668d438 100644 --- a/dht/dht-in.hpp +++ b/dht/dht-in.hpp @@ -179,6 +179,7 @@ class DhtMemberImpl : public DhtMember { void get_value(DhtKey key, td::Promise result) override { get_value_in(key.compute_key_id(), std::move(result)); } + void get_value_many(DhtKey key, std::function callback, td::Promise promise) override; void alarm() override { alarm_timestamp() = td::Timestamp::in(1.0); diff --git a/dht/dht-query.cpp b/dht/dht-query.cpp index b84ef8c37..3d43b1069 100644 --- a/dht/dht-query.cpp +++ b/dht/dht-query.cpp @@ -210,8 +210,11 @@ void DhtQueryFindValue::on_result(td::Result R, adnl::AdnlNodeI send_get_nodes = true; return; } - promise_.set_value(std::move(value)); - need_stop = true; + if (on_value_found(std::move(value))) { + send_get_nodes = true; + } else { + need_stop = true; + } }, [&](ton_api::dht_valueNotFound &v) { add_nodes(DhtNodesList{std::move(v.nodes_), our_network_id()}); @@ -244,7 +247,32 @@ void DhtQueryFindValue::on_result_nodes(td::Result R, adnl::Adn } void DhtQueryFindValue::finish(DhtNodesList list) { - promise_.set_error(td::Status::Error(ErrorCode::notready, "dht key not found")); +} + +bool DhtQueryFindValueSingle::on_value_found(DhtValue value) { + promise_.set_value(std::move(value)); + found_ = true; + return false; +} + +void DhtQueryFindValueSingle::tear_down() { + if (!found_) { + promise_.set_error(td::Status::Error(ErrorCode::notready, "dht key not found")); + } +} + +bool DhtQueryFindValueMany::on_value_found(DhtValue value) { + callback_(std::move(value)); + found_ = true; + return true; +} + +void DhtQueryFindValueMany::tear_down() { + if (found_) { + promise_.set_value(td::Unit()); + } else { + promise_.set_error(td::Status::Error(ErrorCode::notready, "dht key not found")); + } } DhtQueryStore::DhtQueryStore(DhtValue key_value, DhtMember::PrintId print_id, adnl::AdnlNodeIdShort src, diff --git a/dht/dht-query.hpp b/dht/dht-query.hpp index e47403618..e305f107a 100644 --- a/dht/dht-query.hpp +++ b/dht/dht-query.hpp @@ -126,16 +126,11 @@ class DhtQueryFindNodes : public DhtQuery { }; class DhtQueryFindValue : public DhtQuery { - private: - td::Promise promise_; - public: DhtQueryFindValue(DhtKeyId key, DhtMember::PrintId print_id, adnl::AdnlNodeIdShort src, DhtNodesList list, td::uint32 k, td::uint32 a, td::int32 our_network_id, DhtNode self, bool client_only, - td::actor::ActorId node, td::actor::ActorId adnl, - td::Promise promise) - : DhtQuery(key, print_id, src, k, a, our_network_id, std::move(self), client_only, node, adnl) - , promise_(std::move(promise)) { + td::actor::ActorId node, td::actor::ActorId adnl) + : DhtQuery(key, print_id, src, k, a, our_network_id, std::move(self), client_only, node, adnl) { add_nodes(std::move(list)); } void send_one_query(adnl::AdnlNodeIdShort id) override; @@ -146,6 +141,48 @@ class DhtQueryFindValue : public DhtQuery { std::string get_name() const override { return "find value"; } + + virtual bool on_value_found(DhtValue value) = 0; +}; + +class DhtQueryFindValueSingle : public DhtQueryFindValue { + public: + DhtQueryFindValueSingle(DhtKeyId key, DhtMember::PrintId print_id, adnl::AdnlNodeIdShort src, DhtNodesList list, + td::uint32 k, td::uint32 a, td::int32 our_network_id, DhtNode self, bool client_only, + td::actor::ActorId node, td::actor::ActorId adnl, + td::Promise promise) + : DhtQueryFindValue(key, print_id, src, std::move(list), k, a, our_network_id, std::move(self), client_only, node, + adnl) + , promise_(std::move(promise)) { + add_nodes(std::move(list)); + } + bool on_value_found(DhtValue value) override; + void tear_down() override; + + private: + td::Promise promise_; + bool found_ = false; +}; + +class DhtQueryFindValueMany : public DhtQueryFindValue { + public: + DhtQueryFindValueMany(DhtKeyId key, DhtMember::PrintId print_id, adnl::AdnlNodeIdShort src, DhtNodesList list, + td::uint32 k, td::uint32 a, td::int32 our_network_id, DhtNode self, bool client_only, + td::actor::ActorId node, td::actor::ActorId adnl, + std::function callback, td::Promise promise) + : DhtQueryFindValue(key, print_id, src, std::move(list), k, a, our_network_id, std::move(self), client_only, node, + adnl) + , callback_(std::move(callback)) + , promise_(std::move(promise)) { + add_nodes(std::move(list)); + } + bool on_value_found(DhtValue value) override; + void tear_down() override; + + private: + std::function callback_; + td::Promise promise_; + bool found_ = false; }; class DhtQueryStore : public td::actor::Actor { diff --git a/dht/dht.cpp b/dht/dht.cpp index 8d7b02b7d..b774b5f91 100644 --- a/dht/dht.cpp +++ b/dht/dht.cpp @@ -470,7 +470,7 @@ void DhtMemberImpl::get_value_in(DhtKeyId key, td::Promise result) { network_id = network_id_, id = id_, client_only = client_only_](td::Result R) mutable { R.ensure(); - td::actor::create_actor("FindValueQuery", key, print_id, id, std::move(list), k, a, network_id, + td::actor::create_actor("FindValueQuery", key, print_id, id, std::move(list), k, a, network_id, R.move_as_ok(), client_only, SelfId, adnl, std::move(promise)) .release(); }); @@ -478,6 +478,21 @@ void DhtMemberImpl::get_value_in(DhtKeyId key, td::Promise result) { get_self_node(std::move(P)); } +void DhtMemberImpl::get_value_many(DhtKey key, std::function callback, td::Promise promise) { + DhtKeyId key_id = key.compute_key_id(); + auto P = td::PromiseCreator::lambda( + [key = key_id, callback = std::move(callback), promise = std::move(promise), SelfId = actor_id(this), + print_id = print_id(), adnl = adnl_, list = get_nearest_nodes(key_id, k_ * 2), k = k_, a = a_, + network_id = network_id_, id = id_, client_only = client_only_](td::Result R) mutable { + R.ensure(); + td::actor::create_actor("FindValueManyQuery", key, print_id, id, std::move(list), k, a, + network_id, R.move_as_ok(), client_only, SelfId, adnl, + std::move(callback), std::move(promise)) + .release(); + }); + get_self_node(std::move(P)); +} + void DhtMemberImpl::register_reverse_connection(adnl::AdnlNodeIdFull client, td::Promise promise) { auto client_short = client.compute_short_id(); td::uint32 ttl = (td::uint32)td::Clocks::system() + 300; diff --git a/dht/dht.h b/dht/dht.h index b9c65c8a7..5abff94a1 100644 --- a/dht/dht.h +++ b/dht/dht.h @@ -53,6 +53,7 @@ class Dht : public td::actor::Actor { virtual void set_value(DhtValue key_value, td::Promise result) = 0; virtual void get_value(DhtKey key, td::Promise result) = 0; + virtual void get_value_many(DhtKey key, std::function callback, td::Promise promise) = 0; virtual void register_reverse_connection(adnl::AdnlNodeIdFull client, td::Promise promise) = 0; virtual void request_reverse_ping(adnl::AdnlNode target, adnl::AdnlNodeIdShort client, diff --git a/overlay/overlay.cpp b/overlay/overlay.cpp index f964ccc5c..43b3b7e5a 100644 --- a/overlay/overlay.cpp +++ b/overlay/overlay.cpp @@ -283,11 +283,14 @@ void OverlayImpl::alarm() { } if (next_dht_query_ && next_dht_query_.is_in_past()) { next_dht_query_ = td::Timestamp::never(); - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result res) { - td::actor::send_closure(SelfId, &OverlayImpl::receive_dht_nodes, std::move(res), true); - }); - td::actor::send_closure(dht_node_, &dht::Dht::get_value, dht::DhtKey{overlay_id_.pubkey_hash(), "nodes", 0}, - std::move(P)); + std::function callback = [SelfId = actor_id(this)](dht::DhtValue value) { + td::actor::send_closure(SelfId, &OverlayImpl::receive_dht_nodes, std::move(value)); + }; + td::Promise on_finish = [SelfId = actor_id(this)](td::Result R) { + td::actor::send_closure(SelfId, &OverlayImpl::dht_lookup_finished, R.move_as_status()); + }; + td::actor::send_closure(dht_node_, &dht::Dht::get_value_many, dht::DhtKey{overlay_id_.pubkey_hash(), "nodes", 0}, + std::move(callback), std::move(on_finish)); } if (update_db_at_.is_in_past()) { if (peers_.size() > 0) { @@ -311,30 +314,30 @@ void OverlayImpl::alarm() { } } -void OverlayImpl::receive_dht_nodes(td::Result res, bool dummy) { +void OverlayImpl::receive_dht_nodes(dht::DhtValue v) { CHECK(public_); - if (res.is_ok()) { - auto v = res.move_as_ok(); - auto R = fetch_tl_object(v.value().clone(), true); - if (R.is_ok()) { - auto r = R.move_as_ok(); - VLOG(OVERLAY_INFO) << this << ": received " << r->nodes_.size() << " nodes from overlay"; - VLOG(OVERLAY_EXTRA_DEBUG) << this << ": nodes: " << ton_api::to_string(r); - std::vector nodes; - for (auto &n : r->nodes_) { - auto N = OverlayNode::create(n); - if (N.is_ok()) { - nodes.emplace_back(N.move_as_ok()); - } + auto R = fetch_tl_object(v.value().clone(), true); + if (R.is_ok()) { + auto r = R.move_as_ok(); + VLOG(OVERLAY_INFO) << this << ": received " << r->nodes_.size() << " nodes from overlay"; + VLOG(OVERLAY_EXTRA_DEBUG) << this << ": nodes: " << ton_api::to_string(r); + std::vector nodes; + for (auto &n : r->nodes_) { + auto N = OverlayNode::create(n); + if (N.is_ok()) { + nodes.emplace_back(N.move_as_ok()); } - add_peers(std::move(nodes)); - } else { - VLOG(OVERLAY_WARNING) << this << ": incorrect value in DHT for overlay nodes: " << R.move_as_error(); } + add_peers(std::move(nodes)); } else { - VLOG(OVERLAY_NOTICE) << this << ": can not get value from DHT: " << res.move_as_error(); + VLOG(OVERLAY_WARNING) << this << ": incorrect value in DHT for overlay nodes: " << R.move_as_error(); } +} +void OverlayImpl::dht_lookup_finished(td::Status S) { + if (S.is_error()) { + VLOG(OVERLAY_NOTICE) << this << ": can not get value from DHT: " << S; + } if (!(next_dht_store_query_ && next_dht_store_query_.is_in_past())) { finish_dht_query(); return; diff --git a/overlay/overlay.hpp b/overlay/overlay.hpp index 90fcc43d7..8fb3d91d7 100644 --- a/overlay/overlay.hpp +++ b/overlay/overlay.hpp @@ -166,7 +166,8 @@ class OverlayImpl : public Overlay { certs_[key] = std::move(cert); } - void receive_dht_nodes(td::Result res, bool dummy); + void receive_dht_nodes(dht::DhtValue v); + void dht_lookup_finished(td::Status S); void update_dht_nodes(OverlayNode node); void update_neighbours(td::uint32 nodes_to_change); diff --git a/tdutils/td/utils/Status.h b/tdutils/td/utils/Status.h index 9c4f1b7df..8bc210dba 100644 --- a/tdutils/td/utils/Status.h +++ b/tdutils/td/utils/Status.h @@ -555,6 +555,12 @@ class Result { }; return status_.move_as_error_suffix(suffix); } + Status move_as_status() TD_WARN_UNUSED_RESULT { + if (status_.is_error()) { + return move_as_error(); + } + return Status::OK(); + } const T &ok() const { LOG_CHECK(status_.is_ok()) << status_; return value_; From 622dc8676ade370f84a241d621c13b84dfa247b6 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 16 Aug 2024 10:23:41 +0300 Subject: [PATCH 17/37] Limit the number of threads to 127 (#1111) --- dht-server/dht-server.cpp | 12 ++++++++---- validator-engine/validator-engine.cpp | 12 ++++++++---- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/dht-server/dht-server.cpp b/dht-server/dht-server.cpp index 025cf7d51..eb183cad6 100644 --- a/dht-server/dht-server.cpp +++ b/dht-server/dht-server.cpp @@ -1231,15 +1231,19 @@ int main(int argc, char *argv[]) { }); td::uint32 threads = 7; p.add_checked_option( - 't', "threads", PSTRING() << "number of threads (default=" << threads << ")", [&](td::Slice fname) { + 't', "threads", PSTRING() << "number of threads (default=" << threads << ")", [&](td::Slice arg) { td::int32 v; try { - v = std::stoi(fname.str()); + v = std::stoi(arg.str()); } catch (...) { return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: not a number"); } - if (v < 1 || v > 256) { - return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: should be in range [1..256]"); + if (v <= 0) { + return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: should be > 0"); + } + if (v > 127) { + LOG(WARNING) << "`--threads " << v << "` is too big, effective value will be 127"; + v = 127; } threads = v; return td::Status::OK(); diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 88cef8d49..a9a3b21d4 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -4157,15 +4157,19 @@ int main(int argc, char *argv[]) { }); td::uint32 threads = 7; p.add_checked_option( - 't', "threads", PSTRING() << "number of threads (default=" << threads << ")", [&](td::Slice fname) { + 't', "threads", PSTRING() << "number of threads (default=" << threads << ")", [&](td::Slice arg) { td::int32 v; try { - v = std::stoi(fname.str()); + v = std::stoi(arg.str()); } catch (...) { return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: not a number"); } - if (v < 1 || v > 256) { - return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: should be in range [1..256]"); + if (v <= 0) { + return td::Status::Error(ton::ErrorCode::error, "bad value for --threads: should be > 0"); + } + if (v > 127) { + LOG(WARNING) << "`--threads " << v << "` is too big, effective value will be 127"; + v = 127; } threads = v; return td::Status::OK(); From b70090dae35c737d2f7246ee44a307d9b565ee4e Mon Sep 17 00:00:00 2001 From: krigga Date: Tue, 20 Aug 2024 19:50:59 +0300 Subject: [PATCH 18/37] Disable testing and fuzzing for openssl when building WASM packages (#1116) Co-authored-by: EmelyanenkoK --- assembly/wasm/fift-func-wasm-build-ubuntu.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/assembly/wasm/fift-func-wasm-build-ubuntu.sh b/assembly/wasm/fift-func-wasm-build-ubuntu.sh index e7a54d16f..2d3507b23 100644 --- a/assembly/wasm/fift-func-wasm-build-ubuntu.sh +++ b/assembly/wasm/fift-func-wasm-build-ubuntu.sh @@ -85,7 +85,7 @@ cd .. if [ ! -f "openssl/openssl_em" ]; then cd openssl make clean - emconfigure ./Configure linux-generic32 no-shared no-dso no-engine no-unit-test + emconfigure ./Configure linux-generic32 no-shared no-dso no-engine no-unit-test no-tests no-fuzz-afl no-fuzz-libfuzzer sed -i 's/CROSS_COMPILE=.*/CROSS_COMPILE=/g' Makefile sed -i 's/-ldl//g' Makefile sed -i 's/-O3/-Os/g' Makefile From dc26c3be671b13cd7d07f232df8be4ccb8cdc2e0 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 20 Aug 2024 19:54:16 +0300 Subject: [PATCH 19/37] Improve validator session stats (#1117) * Improve validator session stats * Collator stats: block limits, number of processed external messages * Collator and validator work time * Last key block seqno * Approvers and signers * End validator session stats --- catchain/catchain.h | 1 + catchain/catchain.hpp | 9 ++ tdutils/td/utils/Timer.cpp | 43 +++++++++ tdutils/td/utils/Timer.h | 18 ++++ tl/generate/scheme/ton_api.tl | 25 +++-- tl/generate/scheme/ton_api.tlo | Bin 91004 -> 92472 bytes validator-session/validator-session-types.h | 14 +++ validator-session/validator-session.cpp | 29 ++++++ validator-session/validator-session.h | 1 + validator-session/validator-session.hpp | 1 + validator/impl/collator-impl.h | 5 + validator/impl/collator.cpp | 63 ++++++++++++ validator/impl/validate-query.cpp | 21 ++++ validator/impl/validate-query.hpp | 4 + validator/interfaces/validator-manager.h | 17 ++++ validator/manager-disk.hpp | 3 + validator/manager-hardfork.hpp | 3 + validator/manager.cpp | 102 +++++++++++++++++--- validator/manager.hpp | 19 +++- validator/validator-group.cpp | 11 +++ validator/validator-group.hpp | 11 ++- 21 files changed, 373 insertions(+), 27 deletions(-) diff --git a/catchain/catchain.h b/catchain/catchain.h index 912957e56..c5c8af28d 100644 --- a/catchain/catchain.h +++ b/catchain/catchain.h @@ -96,6 +96,7 @@ class CatChain : public td::actor::Actor { virtual void send_query_via(const PublicKeyHash &dst, std::string name, td::Promise promise, td::Timestamp timeout, td::BufferSlice query, td::uint64 max_answer_size, td::actor::ActorId via) = 0; + virtual void get_source_heights(td::Promise> promise) = 0; virtual void destroy() = 0; static td::actor::ActorOwn create(std::unique_ptr callback, const CatChainOptions &opts, diff --git a/catchain/catchain.hpp b/catchain/catchain.hpp index 8c8bb99ae..586cf4744 100644 --- a/catchain/catchain.hpp +++ b/catchain/catchain.hpp @@ -115,6 +115,15 @@ class CatChainImpl : public CatChain { td::actor::send_closure(receiver_, &CatChainReceiverInterface::send_custom_query_data_via, dst, name, std::move(promise), timeout, std::move(query), max_answer_size, via); } + void get_source_heights(td::Promise> promise) override { + std::vector heights(top_source_blocks_.size(), 0); + for (size_t i = 0; i < top_source_blocks_.size(); ++i) { + if (top_source_blocks_[i]) { + heights[i] = top_source_blocks_[i]->height(); + } + } + promise.set_result(std::move(heights)); + } void destroy() override; CatChainImpl(std::unique_ptr callback, const CatChainOptions &opts, td::actor::ActorId keyring, td::actor::ActorId adnl, diff --git a/tdutils/td/utils/Timer.cpp b/tdutils/td/utils/Timer.cpp index 1f72fba96..24de099aa 100644 --- a/tdutils/td/utils/Timer.cpp +++ b/tdutils/td/utils/Timer.cpp @@ -91,4 +91,47 @@ double PerfWarningTimer::elapsed() const { return Time::now() - start_at_; } +static double thread_cpu_clock() { +#if defined(CLOCK_THREAD_CPUTIME_ID) + timespec ts; + int result = clock_gettime(CLOCK_THREAD_CPUTIME_ID, &ts); + CHECK(result == 0); + return (double)ts.tv_sec + (double)ts.tv_nsec * 1e-9; +#else + return 0.0; // TODO: MacOS and Windows support (currently cpu timer is used only in validators) +#endif +} + +ThreadCpuTimer::ThreadCpuTimer(bool is_paused) : is_paused_(is_paused) { + if (is_paused_) { + start_time_ = 0; + } else { + start_time_ = thread_cpu_clock(); + } +} + +void ThreadCpuTimer::pause() { + if (is_paused_) { + return; + } + elapsed_ += thread_cpu_clock() - start_time_; + is_paused_ = true; +} + +void ThreadCpuTimer::resume() { + if (!is_paused_) { + return; + } + start_time_ = thread_cpu_clock(); + is_paused_ = false; +} + +double ThreadCpuTimer::elapsed() const { + double res = elapsed_; + if (!is_paused_) { + res += thread_cpu_clock() - start_time_; + } + return res; +} + } // namespace td diff --git a/tdutils/td/utils/Timer.h b/tdutils/td/utils/Timer.h index 3e0cafbf5..a27cac8a7 100644 --- a/tdutils/td/utils/Timer.h +++ b/tdutils/td/utils/Timer.h @@ -62,4 +62,22 @@ class PerfWarningTimer { std::function callback_; }; +class ThreadCpuTimer { + public: + ThreadCpuTimer() : ThreadCpuTimer(false) { + } + explicit ThreadCpuTimer(bool is_paused); + ThreadCpuTimer(const ThreadCpuTimer &other) = default; + ThreadCpuTimer &operator=(const ThreadCpuTimer &other) = default; + + double elapsed() const; + void pause(); + void resume(); + + private: + double elapsed_{0}; + double start_time_; + bool is_paused_{false}; +}; + } // namespace td diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index b33ca5425..bf919b0fd 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -388,8 +388,8 @@ tonNode.newShardBlock block:tonNode.blockIdExt cc_seqno:int data:bytes = tonNode tonNode.blockBroadcastCompressed.data signatures:(vector tonNode.blockSignature) proof_data:bytes = tonNode.blockBroadcaseCompressed.Data; -tonNode.blockBroadcast id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int - signatures:(vector tonNode.blockSignature) +tonNode.blockBroadcast id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int + signatures:(vector tonNode.blockSignature) proof:bytes data:bytes = tonNode.Broadcast; tonNode.blockBroadcastCompressed id:tonNode.blockIdExt catchain_seqno:int validator_set_hash:int flags:# compressed:bytes = tonNode.Broadcast; @@ -769,13 +769,19 @@ http.server.config dhs:(vector http.server.dnsEntry) local_hosts:(vector http.se ---types--- -validatorSession.statsProducer id:int256 candidate_id:int256 block_status:int comment:string - block_timestamp:double is_accepted:Bool is_ours:Bool got_submit_at:double +validatorSession.collationStats bytes:int gas:int lt_delta:int cat_bytes:int cat_gas:int cat_lt_delta:int + limits_log:string ext_msgs_total:int ext_msgs_filtered:int ext_msgs_accepted:int ext_msgs_rejected:int = validadorSession.CollationStats; + +validatorSession.statsProducer id:int256 candidate_id:int256 block_status:int root_hash:int256 file_hash:int256 + comment:string block_timestamp:double is_accepted:Bool is_ours:Bool got_submit_at:double collation_time:double collated_at:double collation_cached:Bool + collation_work_time:double collation_cpu_work_time:double + collation_stats:validatorSession.collationStats validation_time:double validated_at:double validation_cached:Bool + validation_work_time:double validation_cpu_work_time:double gen_utime:double - approved_weight:long approved_33pct_at:double approved_66pct_at:double - signed_weight:long signed_33pct_at:double signed_66pct_at:double + approved_weight:long approved_33pct_at:double approved_66pct_at:double approvers:string + signed_weight:long signed_33pct_at:double signed_66pct_at:double signers:string serialize_time:double deserialize_time:double serialized_size:int = validatorSession.StatsProducer; validatorSession.statsRound timestamp:double producers:(vector validatorSession.statsProducer) = validatorSession.StatsRound; @@ -786,9 +792,14 @@ validatorSession.stats success:Bool id:tonNode.blockIdExt timestamp:double self: first_round:int rounds:(vector validatorSession.statsRound) = validatorSession.Stats; validatorSession.newValidatorGroupStats.node id:int256 weight:long = validatorSession.newValidatorGroupStats.Node; -validatorSession.newValidatorGroupStats session_id:int256 workchain:int shard:long cc_seqno:int timestamp:double +validatorSession.newValidatorGroupStats session_id:int256 workchain:int shard:long cc_seqno:int + last_key_block_seqno:int timestamp:double self_idx:int nodes:(vector validatorSession.newValidatorGroupStats.node) = validatorSession.NewValidatorGroupStats; +validatorSession.endValidatorGroupStats.node id:int256 catchain_blocks:int = validatorSession.endValidatorGroupStats.Node; +validatorSession.endValidatorGroupStats session_id:int256 timestamp:double + nodes:(vector validatorSession.endValidatorGroupStats.node) = validatorSession.EndValidatorGroupStats; + ---functions--- ---types--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index da1aa331d2048803f582b99ede705270f28a22ea..337dd071e3f746cd822236e2dcb63c194b379de4 100644 GIT binary patch delta 841 zcmex!jCIE)R^CUm^{p77fNLY~d3}|4min3UWr;bNDTyihMZu}X#hLkedd~SdIf*4e zR&YsTN%6!D5}WVn?_iXi_R1(;9iq4dq1ZJqW%@;aM&Zdn3?(K{F|=S=WW;%J@&&^c zES-~cgeKP*6-c&d`z+GI(3lF;xKM&oWU{2O{Ny)AKNx#AKQ*>L#R_)Q^ouNv!ivbw z014kbCgj5n^nmoU`4q$f(n!LI$;qh&B^bg*sadJX5MlAj2^JhIAonkp_$P=X7(t-{3Pu)C zC{I^#VpN!JAjBvz{e~c;%H$QkDiWYTWQ0auku#ePhi%(Iuv#8|#x<;+`kvg1oKb5kb=%#)B1M@S?Wl%h(?@*^aRL1_o%UXVjUo@boC zRg|$^R0Nt3fXNF`3spvW|5Mycsh;*50=|J@O1?0~Q`Bp4O&r$K2JQ0PuR5U3%*4@@b^8Ht&B@ku%P T$=St|57^3pWtZ1TFnR+3<5Dl9 delta 209 zcmdmSiS^GhR^CUm^{p77fPEwHdHu~220Iuhix^q3>_0DYW^#bh3KmawcJ;}Bj0z_2 zF#f^VwRxk7{i&^27{3TipCZJ#fEA=neezyobxsiL<}o23_UQ*j8MP-@6uC@)Bg&{U zT}Onm0c_g#4I+#mm{>vP$xT-jWfYsfTbz-T6T}DUlbAd)NO}4PamIk{9TJQ>jMFWo p7;C0WNHZ>({y>_sVtR!P&>WEB>3?JxJ0PqMSw@BJE^>@tya3`PN1gxx diff --git a/validator-session/validator-session-types.h b/validator-session/validator-session-types.h index e13c36d24..78a9b2460 100644 --- a/validator-session/validator-session-types.h +++ b/validator-session/validator-session-types.h @@ -77,6 +77,8 @@ struct ValidatorSessionStats { ValidatorSessionCandidateId candidate_id = ValidatorSessionCandidateId::zero(); int block_status = status_none; double block_timestamp = -1.0; + td::Bits256 root_hash = td::Bits256::zero(); + td::Bits256 file_hash = td::Bits256::zero(); std::string comment; bool is_accepted = false; @@ -159,11 +161,23 @@ struct NewValidatorGroupStats { ValidatorSessionId session_id = ValidatorSessionId::zero(); ShardIdFull shard{masterchainId}; CatchainSeqno cc_seqno = 0; + BlockSeqno last_key_block_seqno = 0; double timestamp = -1.0; td::uint32 self_idx = 0; std::vector nodes; }; +struct EndValidatorGroupStats { + struct Node { + PublicKeyHash id = PublicKeyHash::zero(); + td::uint32 catchain_blocks = 0; + }; + + ValidatorSessionId session_id = ValidatorSessionId::zero(); + double timestamp = -1.0; + std::vector nodes; +}; + } // namespace validatorsession } // namespace ton diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index 46dd44403..be5443785 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -270,6 +270,8 @@ void ValidatorSessionImpl::process_broadcast(PublicKeyHash src, td::BufferSlice } stat->deserialize_time = deserialize_time; stat->serialized_size = data.size(); + stat->root_hash = candidate->root_hash_; + stat->file_hash = file_hash; } if ((td::int32)block_round < (td::int32)cur_round_ - MAX_PAST_ROUND_BLOCK || @@ -468,6 +470,8 @@ void ValidatorSessionImpl::generated_block(td::uint32 round, ValidatorSessionCan stat->collated_at = td::Clocks::system(); stat->block_timestamp = td::Clocks::system(); stat->collation_cached = collation_cached; + stat->root_hash = root_hash; + stat->file_hash = file_hash; } if (round != cur_round_) { return; @@ -602,6 +606,8 @@ void ValidatorSessionImpl::try_approve_block(const SentBlock *block) { if (stat->block_timestamp <= 0.0) { stat->block_timestamp = td::Clocks::system(); } + stat->root_hash = B->root_hash_; + stat->file_hash = td::sha256_bits256(B->data_); } auto P = td::PromiseCreator::lambda([round = cur_round_, hash = block_id, root_hash = block->get_root_hash(), @@ -997,6 +1003,29 @@ void ValidatorSessionImpl::get_current_stats(td::Promise promise.set_result(cur_stats_); } +void ValidatorSessionImpl::get_end_stats(td::Promise promise) { + if (!started_) { + promise.set_error(td::Status::Error(ErrorCode::notready, "not started")); + return; + } + EndValidatorGroupStats stats; + stats.session_id = unique_hash_; + stats.timestamp = td::Clocks::system(); + stats.nodes.resize(description().get_total_nodes()); + for (size_t i = 0; i < stats.nodes.size(); ++i) { + stats.nodes[i].id = description().get_source_id(i); + } + td::actor::send_closure(catchain_, &catchain::CatChain::get_source_heights, + [promise = std::move(promise), + stats = std::move(stats)](td::Result> R) mutable { + TRY_RESULT_PROMISE(promise, heights, std::move(R)); + for (size_t i = 0; i < std::min(heights.size(), stats.nodes.size()); ++i) { + stats.nodes[i].catchain_blocks = heights[i]; + } + promise.set_result(std::move(stats)); + }); +} + void ValidatorSessionImpl::get_validator_group_info_for_litequery( td::uint32 cur_round, td::Promise>> promise) { diff --git a/validator-session/validator-session.h b/validator-session/validator-session.h index 0870f6718..2e1ed9b13 100644 --- a/validator-session/validator-session.h +++ b/validator-session/validator-session.h @@ -105,6 +105,7 @@ class ValidatorSession : public td::actor::Actor { virtual void start() = 0; virtual void destroy() = 0; virtual void get_current_stats(td::Promise promise) = 0; + virtual void get_end_stats(td::Promise promise) = 0; virtual void get_validator_group_info_for_litequery( td::uint32 cur_round, td::Promise>> promise) = 0; diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp index 580582824..2ee4885b9 100644 --- a/validator-session/validator-session.hpp +++ b/validator-session/validator-session.hpp @@ -187,6 +187,7 @@ class ValidatorSessionImpl : public ValidatorSession { void start() override; void destroy() override; void get_current_stats(td::Promise promise) override; + void get_end_stats(td::Promise promise) override; void get_validator_group_info_for_litequery( td::uint32 cur_round, td::Promise>> promise) override; diff --git a/validator/impl/collator-impl.h b/validator/impl/collator-impl.h index 913a0ed87..b8d9e56d3 100644 --- a/validator/impl/collator-impl.h +++ b/validator/impl/collator-impl.h @@ -356,6 +356,11 @@ class Collator final : public td::actor::Actor { public: static td::uint32 get_skip_externals_queue_size(); + + private: + td::Timer work_timer_{true}; + td::ThreadCpuTimer cpu_work_timer_{true}; + CollationStats stats_; }; } // namespace validator diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index c6dd7caf2..f465c0f55 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -1772,6 +1772,12 @@ bool Collator::register_shard_block_creators(std::vector creator_li * @returns True if collation is successful, false otherwise. */ bool Collator::try_collate() { + work_timer_.resume(); + cpu_work_timer_.resume(); + SCOPE_EXIT { + work_timer_.pause(); + cpu_work_timer_.pause(); + }; if (!preinit_complete) { LOG(WARNING) << "running do_preinit()"; if (!do_preinit()) { @@ -3481,6 +3487,29 @@ bool Collator::process_inbound_message(Ref enq_msg, ton::LogicalT return true; } +/** + * Creates a string that explains which limit is exceeded. Used for collator stats. + * + * @param block_limit_status Status of block limits. + * @param cls Which limit class is exceeded. + * + * @returns String for collator stats. + */ +static std::string block_full_comment(const block::BlockLimitStatus& block_limit_status, unsigned cls) { + auto bytes = block_limit_status.estimate_block_size(); + if (!block_limit_status.limits.bytes.fits(cls, bytes)) { + return PSTRING() << "block_full bytes " << bytes; + } + if (!block_limit_status.limits.gas.fits(cls, block_limit_status.gas_used)) { + return PSTRING() << "block_full gas " << block_limit_status.gas_used; + } + auto lt_delta = block_limit_status.cur_lt - block_limit_status.limits.start_lt; + if (!block_limit_status.limits.lt_delta.fits(cls, lt_delta)) { + return PSTRING() << "block_full lt_delta " << lt_delta; + } + return ""; +} + /** * Processes inbound internal messages from message queues of the neighbors. * Messages are processed until the normal limit is reached, soft timeout is reached or there are no more messages. @@ -3495,11 +3524,14 @@ bool Collator::process_inbound_internal_messages() { block_full_ = !block_limit_status_->fits(block::ParamLimits::cl_normal); if (block_full_) { LOG(INFO) << "BLOCK FULL, stop processing inbound internal messages"; + stats_.limits_log += PSTRING() << "INBOUND_INT_MESSAGES: " + << block_full_comment(*block_limit_status_, block::ParamLimits::cl_normal) << "\n"; break; } if (soft_timeout_.is_in_past(td::Timestamp::now())) { block_full_ = true; LOG(WARNING) << "soft timeout reached, stop processing inbound internal messages"; + stats_.limits_log += PSTRING() << "INBOUND_INT_MESSAGES: timeout\n"; break; } auto kv = nb_out_msgs_->extract_cur(); @@ -3547,15 +3579,23 @@ bool Collator::process_inbound_external_messages() { } if (full) { LOG(INFO) << "BLOCK FULL, stop processing external messages"; + stats_.limits_log += PSTRING() << "INBOUND_EXT_MESSAGES: " + << block_full_comment(*block_limit_status_, block::ParamLimits::cl_soft) << "\n"; break; } if (medium_timeout_.is_in_past(td::Timestamp::now())) { LOG(WARNING) << "medium timeout reached, stop processing inbound external messages"; + stats_.limits_log += PSTRING() << "INBOUND_EXT_MESSAGES: timeout\n"; break; } auto ext_msg = ext_msg_struct.cell; ton::Bits256 hash{ext_msg->get_hash().bits()}; int r = process_external_message(std::move(ext_msg)); + if (r > 0) { + ++stats_.ext_msgs_accepted; + } else { + ++stats_.ext_msgs_rejected; + } if (r < 0) { bad_ext_msgs_.emplace_back(ext_msg_struct.hash); return false; @@ -3661,11 +3701,15 @@ bool Collator::process_dispatch_queue() { block_full_ = !block_limit_status_->fits(block::ParamLimits::cl_normal); if (block_full_) { LOG(INFO) << "BLOCK FULL, stop processing dispatch queue"; + stats_.limits_log += PSTRING() << "DISPATCH_QUEUE_STAGE_" << iter << ": " + << block_full_comment(*block_limit_status_, block::ParamLimits::cl_normal) + << "\n"; return true; } if (soft_timeout_.is_in_past(td::Timestamp::now())) { block_full_ = true; LOG(WARNING) << "soft timeout reached, stop processing dispatch queue"; + stats_.limits_log += PSTRING() << "DISPATCH_QUEUE_STAGE_" << iter << ": timeout\n"; return true; } StdSmcAddress src_addr; @@ -3715,6 +3759,7 @@ bool Collator::process_dispatch_queue() { ++total_count; if (total_count >= max_total_count[iter]) { dispatch_queue_total_limit_reached_ = true; + stats_.limits_log += PSTRING() << "DISPATCH_QUEUE_STAGE_" << iter << ": total limit reached\n"; break; } } @@ -4064,6 +4109,8 @@ bool Collator::process_new_messages(bool enqueue_only) { if ((block_full_ || have_unprocessed_account_dispatch_queue_) && !enqueue_only) { LOG(INFO) << "BLOCK FULL, enqueue all remaining new messages"; enqueue_only = true; + stats_.limits_log += PSTRING() << "NEW_MESSAGES: " + << block_full_comment(*block_limit_status_, block::ParamLimits::cl_normal) << "\n"; } LOG(DEBUG) << "have message with lt=" << msg.lt; int res = process_one_new_message(std::move(msg), enqueue_only); @@ -4072,6 +4119,8 @@ bool Collator::process_new_messages(bool enqueue_only) { } else if (res == 3) { LOG(INFO) << "All remaining new messages must be enqueued (BLOCK FULL)"; enqueue_only = true; + stats_.limits_log += PSTRING() << "NEW_MESSAGES: " + << block_full_comment(*block_limit_status_, block::ParamLimits::cl_normal) << "\n"; } } return true; @@ -5435,6 +5484,18 @@ bool Collator::create_block_candidate() { td::actor::send_closure_later(manager, &ValidatorManager::complete_external_messages, std::move(delay_ext_msgs_), std::move(bad_ext_msgs_)); } + + double work_time = work_timer_.elapsed(); + double cpu_work_time = cpu_work_timer_.elapsed(); + LOG(WARNING) << "Collate query work time = " << work_time << "s, cpu time = " << cpu_work_time << "s"; + stats_.bytes = block_limit_status_->estimate_block_size(); + stats_.gas = block_limit_status_->gas_used; + stats_.lt_delta = block_limit_status_->cur_lt - block_limit_status_->limits.start_lt; + stats_.cat_bytes = block_limit_status_->limits.classify_size(stats_.bytes); + stats_.cat_gas = block_limit_status_->limits.classify_gas(stats_.gas); + stats_.cat_lt_delta = block_limit_status_->limits.classify_lt(block_limit_status_->cur_lt); + td::actor::send_closure(manager, &ValidatorManager::record_collate_query_stats, block_candidate->id, work_time, + cpu_work_time, std::move(stats_)); return true; } @@ -5539,6 +5600,7 @@ void Collator::after_get_external_messages(td::Result ext_msg_cell = ext_msg->root_cell(); @@ -5550,6 +5612,7 @@ void Collator::after_get_external_messages(td::Resulthash()); } } diff --git a/validator/impl/validate-query.cpp b/validator/impl/validate-query.cpp index 8c39a1ab4..003b7f9f7 100644 --- a/validator/impl/validate-query.cpp +++ b/validator/impl/validate-query.cpp @@ -118,6 +118,7 @@ bool ValidateQuery::reject_query(std::string error, td::BufferSlice reason) { error = error_ctx() + error; LOG(ERROR) << "REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error; if (main_promise) { + record_stats(); errorlog::ErrorLog::log(PSTRING() << "REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error << ": data=" << block_candidate.id.file_hash.to_hex() << " collated_data=" << block_candidate.collated_file_hash.to_hex()); @@ -155,6 +156,7 @@ bool ValidateQuery::soft_reject_query(std::string error, td::BufferSlice reason) error = error_ctx() + error; LOG(ERROR) << "SOFT REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error; if (main_promise) { + record_stats(); errorlog::ErrorLog::log(PSTRING() << "SOFT REJECT: aborting validation of block candidate for " << shard_.to_str() << " : " << error << ": data=" << block_candidate.id.file_hash.to_hex() << " collated_data=" << block_candidate.collated_file_hash.to_hex()); @@ -177,6 +179,7 @@ bool ValidateQuery::fatal_error(td::Status error) { error.ensure_error(); LOG(ERROR) << "aborting validation of block candidate for " << shard_.to_str() << " : " << error.to_string(); if (main_promise) { + record_stats(); auto c = error.code(); if (c <= -667 && c >= -670) { errorlog::ErrorLog::log(PSTRING() << "FATAL ERROR: aborting validation of block candidate for " << shard_.to_str() @@ -234,6 +237,7 @@ bool ValidateQuery::fatal_error(std::string err_msg, int err_code) { */ void ValidateQuery::finish_query() { if (main_promise) { + record_stats(); LOG(WARNING) << "validate query done"; main_promise.set_result(now_); } @@ -6764,6 +6768,12 @@ bool ValidateQuery::try_validate() { if (pending) { return true; } + work_timer_.resume(); + cpu_work_timer_.resume(); + SCOPE_EXIT { + work_timer_.pause(); + cpu_work_timer_.pause(); + }; try { if (!stage_) { LOG(WARNING) << "try_validate stage 0"; @@ -6903,6 +6913,17 @@ void ValidateQuery::written_candidate() { finish_query(); } +/** + * Sends validation work time to manager. + */ +void ValidateQuery::record_stats() { + double work_time = work_timer_.elapsed(); + double cpu_work_time = cpu_work_timer_.elapsed(); + LOG(WARNING) << "Validate query work time = " << work_time << "s, cpu time = " << cpu_work_time << "s"; + td::actor::send_closure(manager, &ValidatorManager::record_validate_query_stats, block_candidate.id, work_time, + cpu_work_time); +} + } // namespace validator } // namespace ton diff --git a/validator/impl/validate-query.hpp b/validator/impl/validate-query.hpp index 824afb49d..104950938 100644 --- a/validator/impl/validate-query.hpp +++ b/validator/impl/validate-query.hpp @@ -398,6 +398,10 @@ class ValidateQuery : public td::actor::Actor { } return true; } + + td::Timer work_timer_{true}; + td::ThreadCpuTimer cpu_work_timer_{true}; + void record_stats(); }; } // namespace validator diff --git a/validator/interfaces/validator-manager.h b/validator/interfaces/validator-manager.h index 0e9fab73b..b6016bc2b 100644 --- a/validator/interfaces/validator-manager.h +++ b/validator/interfaces/validator-manager.h @@ -52,6 +52,16 @@ struct AsyncSerializerState { UnixTime last_written_block_ts; }; +struct CollationStats { + td::uint32 bytes, gas, lt_delta; + int cat_bytes, cat_gas, cat_lt_delta; + std::string limits_log; + td::uint32 ext_msgs_total = 0; + td::uint32 ext_msgs_filtered = 0; + td::uint32 ext_msgs_accepted = 0; + td::uint32 ext_msgs_rejected = 0; +}; + using ValidateCandidateResult = td::Variant; class ValidatorManager : public ValidatorManagerInterface { @@ -173,6 +183,7 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) = 0; virtual void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) = 0; + virtual void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) = 0; virtual void get_block_handle_for_litequery(BlockIdExt block_id, td::Promise promise) = 0; virtual void get_block_data_for_litequery(BlockIdExt block_id, td::Promise> promise) = 0; @@ -192,6 +203,12 @@ class ValidatorManager : public ValidatorManagerInterface { virtual void add_lite_query_stats(int lite_query_id) { } + virtual void record_collate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, + CollationStats stats) { + } + virtual void record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) { + } + static bool is_persistent_state(UnixTime ts, UnixTime prev_ts) { return ts / (1 << 17) != prev_ts / (1 << 17); } diff --git a/validator/manager-disk.hpp b/validator/manager-disk.hpp index a77be2725..3a77f2301 100644 --- a/validator/manager-disk.hpp +++ b/validator/manager-disk.hpp @@ -388,6 +388,9 @@ class ValidatorManagerImpl : public ValidatorManager { void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override { UNREACHABLE(); } + void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) override { + UNREACHABLE(); + } void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { queue_size_counter_ = diff --git a/validator/manager-hardfork.hpp b/validator/manager-hardfork.hpp index e7175b77b..cf4d3799f 100644 --- a/validator/manager-hardfork.hpp +++ b/validator/manager-hardfork.hpp @@ -450,6 +450,9 @@ class ValidatorManagerImpl : public ValidatorManager { void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override { UNREACHABLE(); } + void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) override { + UNREACHABLE(); + } void get_out_msg_queue_size(BlockIdExt block_id, td::Promise promise) override { if (queue_size_counter_.empty()) { queue_size_counter_ = diff --git a/validator/manager.cpp b/validator/manager.cpp index eb082d91e..fa592a788 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -44,6 +44,7 @@ #include "td/utils/JsonBuilder.h" #include "common/delay.h" +#include "td/utils/filesystem.h" #include "validator/stats-merger.h" @@ -2044,7 +2045,7 @@ void ValidatorManagerImpl::update_shards() { } new_validator_groups_.emplace(val_group_id, std::move(it2->second)); } else { - auto G = create_validator_group(val_group_id, shard, val_set, opts, started_); + auto G = create_validator_group(val_group_id, shard, val_set, key_seqno, opts, started_); if (!G.empty()) { td::actor::send_closure(G, &ValidatorGroup::start, prev, last_masterchain_block_id_, last_masterchain_state_->get_unix_time()); @@ -2100,7 +2101,7 @@ void ValidatorManagerImpl::update_shards() { } new_validator_groups_.emplace(val_group_id, std::move(it2->second)); } else { - auto G = create_validator_group(val_group_id, shard, val_set, opts, started_); + auto G = create_validator_group(val_group_id, shard, val_set, key_seqno, opts, started_); if (!G.empty()) { td::actor::send_closure(G, &ValidatorGroup::start, prev, last_masterchain_block_id_, last_masterchain_state_->get_unix_time()); @@ -2127,7 +2128,7 @@ void ValidatorManagerImpl::update_shards() { } else { new_next_validator_groups_.emplace( val_group_id, - ValidatorGroupEntry{create_validator_group(val_group_id, shard, val_set, opts, started_), shard}); + ValidatorGroupEntry{create_validator_group(val_group_id, shard, val_set, key_seqno, opts, started_), shard}); } } } @@ -2230,7 +2231,7 @@ ValidatorSessionId ValidatorManagerImpl::get_validator_set_id(ShardIdFull shard, } td::actor::ActorOwn ValidatorManagerImpl::create_validator_group( - ValidatorSessionId session_id, ShardIdFull shard, td::Ref validator_set, + ValidatorSessionId session_id, ShardIdFull shard, td::Ref validator_set, BlockSeqno key_seqno, validatorsession::ValidatorSessionOptions opts, bool init_session) { if (check_gc_list_.count(session_id) == 1) { return td::actor::ActorOwn{}; @@ -2241,8 +2242,8 @@ td::actor::ActorOwn ValidatorManagerImpl::create_validator_group auto validator_id = get_validator(shard, validator_set); CHECK(!validator_id.is_zero()); auto G = td::actor::create_actor( - "validatorgroup", shard, validator_id, session_id, validator_set, opts, keyring_, adnl_, rldp_, overlays_, - db_root_, actor_id(this), init_session, + "validatorgroup", shard, validator_id, session_id, validator_set, key_seqno, opts, keyring_, adnl_, rldp_, + overlays_, db_root_, actor_id(this), init_session, opts_->check_unsafe_resync_allowed(validator_set->get_catchain_seqno()), opts_); return G; } @@ -2831,13 +2832,35 @@ void ValidatorManagerImpl::log_validator_session_stats(BlockIdExt block_id, for (const auto &round : stats.rounds) { std::vector> producers; for (const auto &producer : round.producers) { + BlockIdExt cur_block_id{block_id.id, producer.root_hash, producer.file_hash}; + auto it = recorded_block_stats_.find(cur_block_id); + tl_object_ptr collation_stats; + if (it != recorded_block_stats_.end() && it->second.collator_stats_) { + auto &stats = it->second.collator_stats_.value(); + collation_stats = create_tl_object( + stats.bytes, stats.gas, stats.lt_delta, stats.cat_bytes, stats.cat_gas, stats.cat_lt_delta, + stats.limits_log, stats.ext_msgs_total, stats.ext_msgs_filtered, stats.ext_msgs_accepted, + stats.ext_msgs_rejected); + } + std::string approvers, signers; + for (bool x : producer.approvers) { + approvers += (x ? '1' : '0'); + } + for (bool x : producer.signers) { + signers += (x ? '1' : '0'); + } producers.push_back(create_tl_object( - producer.id.bits256_value(), producer.candidate_id, producer.block_status, producer.comment, - producer.block_timestamp, producer.is_accepted, producer.is_ours, producer.got_submit_at, - producer.collation_time, producer.collated_at, producer.collation_cached, producer.validation_time, - producer.validated_at, producer.validation_cached, producer.gen_utime, producer.approved_weight, - producer.approved_33pct_at, producer.approved_66pct_at, producer.signed_weight, producer.signed_33pct_at, - producer.signed_66pct_at, producer.serialize_time, producer.deserialize_time, producer.serialized_size)); + producer.id.bits256_value(), producer.candidate_id, producer.block_status, producer.root_hash, + producer.file_hash, producer.comment, producer.block_timestamp, producer.is_accepted, producer.is_ours, + producer.got_submit_at, producer.collation_time, producer.collated_at, producer.collation_cached, + it == recorded_block_stats_.end() ? -1.0 : it->second.collator_work_time_, + it == recorded_block_stats_.end() ? -1.0 : it->second.collator_cpu_work_time_, std::move(collation_stats), + producer.validation_time, producer.validated_at, producer.validation_cached, + it == recorded_block_stats_.end() ? -1.0 : it->second.validator_work_time_, + it == recorded_block_stats_.end() ? -1.0 : it->second.validator_cpu_work_time_, producer.gen_utime, + producer.approved_weight, producer.approved_33pct_at, producer.approved_66pct_at, std::move(approvers), + producer.signed_weight, producer.signed_33pct_at, producer.signed_66pct_at, std::move(signers), + producer.serialize_time, producer.deserialize_time, producer.serialized_size)); } rounds.push_back(create_tl_object(round.timestamp, std::move(producers))); } @@ -2869,8 +2892,8 @@ void ValidatorManagerImpl::log_new_validator_group_stats(validatorsession::NewVa create_tl_object(node.id.bits256_value(), node.weight)); } auto obj = create_tl_object( - stats.session_id, stats.shard.workchain, stats.shard.shard, stats.cc_seqno, stats.timestamp, stats.self_idx, - std::move(nodes)); + stats.session_id, stats.shard.workchain, stats.shard.shard, stats.cc_seqno, stats.last_key_block_seqno, + stats.timestamp, stats.self_idx, std::move(nodes)); auto s = td::json_encode(td::ToJson(*obj.get()), false); s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); @@ -2879,7 +2902,31 @@ void ValidatorManagerImpl::log_new_validator_group_stats(validatorsession::NewVa file << s << "\n"; file.close(); - LOG(INFO) << "Writing new validator group stats for " << stats.shard.to_str(); + LOG(INFO) << "Writing new validator group stats for " << stats.session_id << " shard=" << stats.shard.to_str() + << " cc_seqno=" << stats.cc_seqno; +} + +void ValidatorManagerImpl::log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) { + std::string fname = opts_->get_session_logs_file(); + if (fname.empty()) { + return; + } + std::vector> nodes; + for (const auto &node : stats.nodes) { + nodes.push_back(create_tl_object(node.id.bits256_value(), + node.catchain_blocks)); + } + auto obj = create_tl_object(stats.session_id, stats.timestamp, + std::move(nodes)); + auto s = td::json_encode(td::ToJson(*obj.get()), false); + s.erase(std::remove_if(s.begin(), s.end(), [](char c) { return c == '\n' || c == '\r'; }), s.end()); + + std::ofstream file; + file.open(fname, std::ios_base::app); + file << s << "\n"; + file.close(); + + LOG(INFO) << "Writing end validator group stats for " << stats.session_id; } void ValidatorManagerImpl::get_block_handle_for_litequery(BlockIdExt block_id, td::Promise promise) { @@ -3165,6 +3212,31 @@ td::actor::ActorOwn ValidatorManagerFactory::create( rldp, overlays); } +void ValidatorManagerImpl::record_collate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, + CollationStats stats) { + auto &record = new_block_stats_record(block_id); + record.collator_work_time_ = work_time; + record.collator_cpu_work_time_ = cpu_work_time; + record.collator_stats_ = std::move(stats); +} + +void ValidatorManagerImpl::record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) { + auto &record = new_block_stats_record(block_id); + record.validator_work_time_ = work_time; + record.validator_cpu_work_time_ = cpu_work_time; +} + +ValidatorManagerImpl::RecordedBlockStats &ValidatorManagerImpl::new_block_stats_record(BlockIdExt block_id) { + if (!recorded_block_stats_.count(block_id)) { + recorded_block_stats_lru_.push(block_id); + if (recorded_block_stats_lru_.size() > 4096) { + recorded_block_stats_.erase(recorded_block_stats_lru_.front()); + recorded_block_stats_lru_.pop(); + } + } + return recorded_block_stats_[block_id]; +} + size_t ValidatorManagerImpl::CheckedExtMsgCounter::get_msg_count(WorkchainId wc, StdSmcAddress addr) { before_query(); auto it1 = counter_cur_.find({wc, addr}); diff --git a/validator/manager.hpp b/validator/manager.hpp index 12354c634..99aa4e0e1 100644 --- a/validator/manager.hpp +++ b/validator/manager.hpp @@ -38,6 +38,7 @@ #include #include #include +#include namespace ton { @@ -261,7 +262,7 @@ class ValidatorManagerImpl : public ValidatorManager { BlockSeqno last_key_block_seqno, const validatorsession::ValidatorSessionOptions &opts); td::actor::ActorOwn create_validator_group(ValidatorSessionId session_id, ShardIdFull shard, - td::Ref validator_set, + td::Ref validator_set, BlockSeqno key_seqno, validatorsession::ValidatorSessionOptions opts, bool create_catchain); struct ValidatorGroupEntry { @@ -589,6 +590,7 @@ class ValidatorManagerImpl : public ValidatorManager { void log_validator_session_stats(BlockIdExt block_id, validatorsession::ValidatorSessionStats stats) override; void log_new_validator_group_stats(validatorsession::NewValidatorGroupStats stats) override; + void log_end_validator_group_stats(validatorsession::EndValidatorGroupStats stats) override; void update_options(td::Ref opts) override; @@ -708,6 +710,21 @@ class ValidatorManagerImpl : public ValidatorManager { td::uint32 ls_stats_check_ext_messages_{0}; td::actor::ActorOwn candidates_buffer_; + + struct RecordedBlockStats { + double collator_work_time_ = -1.0; + double collator_cpu_work_time_ = -1.0; + td::optional collator_stats_; + double validator_work_time_ = -1.0; + double validator_cpu_work_time_ = -1.0; + }; + std::map recorded_block_stats_; + std::queue recorded_block_stats_lru_; + + void record_collate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time, + CollationStats stats) override; + void record_validate_query_stats(BlockIdExt block_id, double work_time, double cpu_work_time) override; + RecordedBlockStats &new_block_stats_record(BlockIdExt block_id); }; } // namespace validator diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index fc3ebe541..4b61c07cd 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -386,6 +386,7 @@ void ValidatorGroup::start(std::vector prev, BlockIdExt min_masterch stats.session_id = session_id_; stats.shard = shard_; stats.cc_seqno = validator_set_->get_catchain_seqno(); + stats.last_key_block_seqno = last_key_block_seqno_; stats.timestamp = td::Clocks::system(); td::uint32 idx = 0; for (const auto& node : validator_set_->export_vector()) { @@ -417,6 +418,16 @@ void ValidatorGroup::destroy() { td::actor::send_closure(manager, &ValidatorManager::log_validator_session_stats, block_id, std::move(stats)); }); + td::actor::send_closure(session_, &validatorsession::ValidatorSession::get_end_stats, + [manager = manager_](td::Result R) { + if (R.is_error()) { + LOG(DEBUG) << "Failed to get validator session end stats: " << R.move_as_error(); + return; + } + auto stats = R.move_as_ok(); + td::actor::send_closure(manager, &ValidatorManager::log_end_validator_group_stats, + std::move(stats)); + }); auto ses = session_.release(); delay_action([ses]() mutable { td::actor::send_closure(ses, &validatorsession::ValidatorSession::destroy); }, td::Timestamp::in(10.0)); diff --git a/validator/validator-group.hpp b/validator/validator-group.hpp index 3499da9d7..936d2fdc7 100644 --- a/validator/validator-group.hpp +++ b/validator/validator-group.hpp @@ -69,15 +69,17 @@ class ValidatorGroup : public td::actor::Actor { } ValidatorGroup(ShardIdFull shard, PublicKeyHash local_id, ValidatorSessionId session_id, - td::Ref validator_set, validatorsession::ValidatorSessionOptions config, - td::actor::ActorId keyring, td::actor::ActorId adnl, - td::actor::ActorId rldp, td::actor::ActorId overlays, - std::string db_root, td::actor::ActorId validator_manager, bool create_session, + td::Ref validator_set, BlockSeqno last_key_block_seqno, + validatorsession::ValidatorSessionOptions config, td::actor::ActorId keyring, + td::actor::ActorId adnl, td::actor::ActorId rldp, + td::actor::ActorId overlays, std::string db_root, + td::actor::ActorId validator_manager, bool create_session, bool allow_unsafe_self_blocks_resync, td::Ref opts) : shard_(shard) , local_id_(std::move(local_id)) , session_id_(session_id) , validator_set_(std::move(validator_set)) + , last_key_block_seqno_(last_key_block_seqno) , config_(std::move(config)) , keyring_(keyring) , adnl_(adnl) @@ -115,6 +117,7 @@ class ValidatorGroup : public td::actor::Actor { UnixTime min_ts_; td::Ref validator_set_; + BlockSeqno last_key_block_seqno_; validatorsession::ValidatorSessionOptions config_; td::actor::ActorId keyring_; From 9803d004c44efcdf55820149270030e91ff57cc6 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 20 Aug 2024 19:55:01 +0300 Subject: [PATCH 20/37] Fix getting creator stats in lite-client (#1115) --- lite-client/lite-client.cpp | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index 020aca705..02a5fab67 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -3429,9 +3429,7 @@ void TestNode::got_creator_stats(ton::BlockIdExt req_blkid, ton::BlockIdExt blki promise.set_error(td::Status::Error(PSLICE() << "invalid CreatorStats record with key " << key.to_hex())); return; } - if (mc_cnt.modified_since(min_utime) || shard_cnt.modified_since(min_utime)) { - func(key, mc_cnt, shard_cnt); - } + func(key, mc_cnt, shard_cnt); allow_eq = false; } if (complete) { From 18305ab2e6767e9de7820cdac5a43ce8cc108eef Mon Sep 17 00:00:00 2001 From: neodix42 Date: Fri, 23 Aug 2024 11:59:40 +0300 Subject: [PATCH 21/37] * update links to docker image (#1109) * include fift and func into Docker image Co-authored-by: EmelyanenkoK Co-authored-by: neodiX --- Dockerfile | 6 +++++- docker/ton-ali.yaml | 2 +- docker/ton-aws.yaml | 2 +- docker/ton-gcp.yaml | 2 +- docker/ton-metal-lb.yaml | 2 +- docker/ton-node-port.yaml | 2 +- 6 files changed, 10 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index cf4187630..f4ea43759 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,7 +27,7 @@ RUN apt-get update && \ apt-get install -y wget curl libatomic1 openssl libsecp256k1-dev libsodium-dev libmicrohttpd-dev liblz4-dev libjemalloc-dev htop net-tools netcat iptraf-ng jq tcpdump pv plzip && \ rm -rf /var/lib/apt/lists/* -RUN mkdir -p /var/ton-work/db /var/ton-work/scripts +RUN mkdir -p /var/ton-work/db /var/ton-work/scripts /usr/share/ton/smartcont/ /usr/lib/fift/ COPY --from=builder /ton/build/storage/storage-daemon/storage-daemon /usr/local/bin/ COPY --from=builder /ton/build/storage/storage-daemon/storage-daemon-cli /usr/local/bin/ @@ -35,6 +35,10 @@ COPY --from=builder /ton/build/lite-client/lite-client /usr/local/bin/ COPY --from=builder /ton/build/validator-engine/validator-engine /usr/local/bin/ COPY --from=builder /ton/build/validator-engine-console/validator-engine-console /usr/local/bin/ COPY --from=builder /ton/build/utils/generate-random-id /usr/local/bin/ +COPY --from=builder /ton/build/crypto/fift /usr/local/bin/ +COPY --from=builder /ton/build/crypto/func /usr/local/bin/ +COPY --from=builder /ton/crypto/smartcont/* /usr/share/ton/smartcont/ +COPY --from=builder /ton/crypto/fift/lib/* /usr/lib/fift/ WORKDIR /var/ton-work/db COPY ./docker/init.sh ./docker/control.template /var/ton-work/scripts/ diff --git a/docker/ton-ali.yaml b/docker/ton-ali.yaml index 03ffbdb0f..2dd5daf6a 100644 --- a/docker/ton-ali.yaml +++ b/docker/ton-ali.yaml @@ -27,7 +27,7 @@ spec: spec: containers: - name: validator-engine-container - image: ghcr.io/neodix42/ton:latest + image: ghcr.io/ton-blockchain/ton:latest env: - name: PUBLIC_IP value: "" diff --git a/docker/ton-aws.yaml b/docker/ton-aws.yaml index da16cbae9..4cab1b556 100644 --- a/docker/ton-aws.yaml +++ b/docker/ton-aws.yaml @@ -27,7 +27,7 @@ spec: spec: containers: - name: validator-engine-container - image: ghcr.io/neodix42/ton:latest + image: ghcr.io/ton-blockchain/ton:latest env: - name: PUBLIC_IP value: "" diff --git a/docker/ton-gcp.yaml b/docker/ton-gcp.yaml index 0ded5a794..e3d89a06e 100644 --- a/docker/ton-gcp.yaml +++ b/docker/ton-gcp.yaml @@ -27,7 +27,7 @@ spec: spec: containers: - name: validator-engine-container - image: ghcr.io/neodix42/ton:latest + image: ghcr.io/ton-blockchain/ton:latest env: - name: PUBLIC_IP value: "" diff --git a/docker/ton-metal-lb.yaml b/docker/ton-metal-lb.yaml index ceaf3a7c0..1e62a45ab 100644 --- a/docker/ton-metal-lb.yaml +++ b/docker/ton-metal-lb.yaml @@ -11,7 +11,7 @@ spec: claimName: validator-engine-pvc containers: - name: validator-engine-container - image: ghcr.io/neodix42/ton:latest + image: ghcr.io/ton-blockchain/ton:latest env: - name: PUBLIC_IP value: "" diff --git a/docker/ton-node-port.yaml b/docker/ton-node-port.yaml index ec594031f..a9f1fe5c8 100644 --- a/docker/ton-node-port.yaml +++ b/docker/ton-node-port.yaml @@ -21,7 +21,7 @@ spec: claimName: validator-engine-pvc containers: - name: validator-engine-container - image: ghcr.io/neodix42/ton:latest + image: ghcr.io/ton-blockchain/ton:latest env: - name: PUBLIC_IP value: "" From 31840a7aa3808c7e40c2b9c77222962561ddbb19 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Mon, 26 Aug 2024 17:53:42 +0300 Subject: [PATCH 22/37] Set default state ttl to 86400, set serializer delay to up to 6h (#1125) --- validator/state-serializer.cpp | 2 +- validator/validator.h | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index b27561b63..f7ea7efe9 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -160,7 +160,7 @@ void AsyncStateSerializer::next_iteration() { LOG(ERROR) << "started serializing persistent state for " << masterchain_handle_->id().id.to_str(); // block next attempts immediately, but send actual request later running_ = true; - double delay = td::Random::fast(0, 3600); + double delay = td::Random::fast(0, 3600 * 6); LOG(WARNING) << "serializer delay = " << delay << "s"; delay_action( [SelfId = actor_id(this)]() { diff --git a/validator/validator.h b/validator/validator.h index 3bceec6fe..bbe22bb43 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -145,7 +145,7 @@ struct ValidatorManagerOptions : public td::CntObject { std::function check_shard = [](ShardIdFull, CatchainSeqno, ShardCheckMode) { return true; }, bool allow_blockchain_init = false, double sync_blocks_before = 3600, double block_ttl = 86400, - double state_ttl = 3600, double archive_ttl = 86400 * 7, double key_proof_ttl = 86400 * 3650, + double state_ttl = 86400, double archive_ttl = 86400 * 7, double key_proof_ttl = 86400 * 3650, double max_mempool_num = 999999, bool initial_sync_disabled = false); }; From b5fd8fa610f6043b2373171f71f469232bb8d415 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Tue, 27 Aug 2024 18:10:17 +0300 Subject: [PATCH 23/37] Disable state serializer on masterchain validators (#1129) --- validator/manager.cpp | 12 ++++++ validator/state-serializer.cpp | 72 ++++++++++++++++++++-------------- validator/state-serializer.hpp | 13 ++++-- 3 files changed, 64 insertions(+), 33 deletions(-) diff --git a/validator/manager.cpp b/validator/manager.cpp index fa592a788..8b7d0eb1d 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2057,6 +2057,7 @@ void ValidatorManagerImpl::update_shards() { } } + bool validating_masterchain = false; if (allow_validate_) { for (auto &desc : new_shards) { auto shard = desc.first; @@ -2073,6 +2074,9 @@ void ValidatorManagerImpl::update_shards() { auto validator_id = get_validator(shard, val_set); if (!validator_id.is_zero()) { + if (shard.is_masterchain()) { + validating_masterchain = true; + } auto val_group_id = get_validator_set_id(shard, val_set, opts_hash, key_seqno, opts); if (force_recover) { @@ -2167,6 +2171,14 @@ void ValidatorManagerImpl::update_shards() { td::actor::send_closure(SelfId, &ValidatorManagerImpl::written_destroyed_validator_sessions, std::move(gc)); }); td::actor::send_closure(db_, &Db::update_destroyed_validator_sessions, gc_list_, std::move(P)); + + if (!serializer_.empty()) { + td::actor::send_closure( + serializer_, &AsyncStateSerializer::auto_disable_serializer, + validating_masterchain && + last_masterchain_state_->get_validator_set(ShardIdFull{masterchainId})->export_vector().size() * 2 <= + last_masterchain_state_->get_total_validator_set(0)->export_vector().size()); + } } } // namespace validator diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index f7ea7efe9..ef79d33cb 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -95,7 +95,8 @@ void AsyncStateSerializer::request_previous_state_files() { } void AsyncStateSerializer::got_previous_state_files(std::vector> files) { - previous_state_files_ = std::move(files); + previous_state_cache_ = std::make_shared(); + previous_state_cache_->state_files = std::move(files); request_masterchain_state(); } @@ -151,7 +152,10 @@ void AsyncStateSerializer::next_iteration() { need_serialize(masterchain_handle_)) { if (!have_masterchain_state_ && !opts_->get_state_serializer_enabled()) { LOG(ERROR) << "skipping serializing persistent state for " << masterchain_handle_->id().id.to_str() - << ": serializer is disabled"; + << ": serializer is disabled (by user)"; + } else if (!have_masterchain_state_ && auto_disabled_) { + LOG(ERROR) << "skipping serializing persistent state for " << masterchain_handle_->id().id.to_str() + << ": serializer is disabled (automatically)"; } else if (!have_masterchain_state_ && have_newer_persistent_state(masterchain_handle_->unix_time())) { LOG(ERROR) << "skipping serializing persistent state for " << masterchain_handle_->id().id.to_str() << ": newer key block with ts=" << last_known_key_block_ts_ << " exists"; @@ -182,9 +186,7 @@ void AsyncStateSerializer::next_iteration() { } last_key_block_ts_ = masterchain_handle_->unix_time(); last_key_block_id_ = masterchain_handle_->id(); - previous_state_files_ = {}; previous_state_cache_ = {}; - previous_state_cur_shards_ = {}; } if (!saved_to_db_) { running_ = true; @@ -252,27 +254,24 @@ class CachedCellDbReader : public vm::CellDbReader { td::uint64 cached_reqs_ = 0; }; -void AsyncStateSerializer::prepare_previous_state_cache(ShardIdFull shard) { - if (!opts_->get_fast_state_serializer_enabled()) { - return; - } +void AsyncStateSerializer::PreviousStateCache::prepare_cache(ShardIdFull shard) { std::vector prev_shards; - for (const auto& [_, prev_shard] : previous_state_files_) { + for (const auto& [_, prev_shard] : state_files) { if (shard_intersects(shard, prev_shard)) { prev_shards.push_back(prev_shard); } } - if (prev_shards == previous_state_cur_shards_) { + if (prev_shards == cur_shards) { return; } - previous_state_cur_shards_ = std::move(prev_shards); - previous_state_cache_ = {}; - if (previous_state_cur_shards_.empty()) { + cur_shards = std::move(prev_shards); + cache = {}; + if (cur_shards.empty()) { return; } td::Timer timer; LOG(WARNING) << "Preloading previous persistent state for shard " << shard.to_str() << " (" - << previous_state_cur_shards_.size() << " files)"; + << cur_shards.size() << " files)"; std::map> cells; std::function)> dfs = [&](td::Ref cell) { td::Bits256 hash = cell->get_hash().bits(); @@ -285,7 +284,7 @@ void AsyncStateSerializer::prepare_previous_state_cache(ShardIdFull shard) { dfs(cs.prefetch_ref(i)); } }; - for (const auto& [file, prev_shard] : previous_state_files_) { + for (const auto& [file, prev_shard] : state_files) { if (!shard_intersects(shard, prev_shard)) { continue; } @@ -300,22 +299,20 @@ void AsyncStateSerializer::prepare_previous_state_cache(ShardIdFull shard) { LOG(WARNING) << "Deserialize error : " << r_root.move_as_error(); continue; } - r_data = {}; + r_data.clear(); dfs(r_root.move_as_ok()); } LOG(WARNING) << "Preloaded previous state: " << cells.size() << " cells in " << timer.elapsed() << "s"; - previous_state_cache_ = std::make_shared>>(std::move(cells)); + cache = std::make_shared>>(std::move(cells)); } void AsyncStateSerializer::got_masterchain_state(td::Ref state, std::shared_ptr cell_db_reader) { - if (!opts_->get_state_serializer_enabled()) { + if (!opts_->get_state_serializer_enabled() || auto_disabled_) { stored_masterchain_state(); return; } LOG(ERROR) << "serializing masterchain state " << masterchain_handle_->id().id.to_str(); - prepare_previous_state_cache(state->get_shard()); - auto new_cell_db_reader = std::make_shared(cell_db_reader, previous_state_cache_); have_masterchain_state_ = true; CHECK(next_idx_ == 0); CHECK(shards_.size() == 0); @@ -325,10 +322,16 @@ void AsyncStateSerializer::got_masterchain_state(td::Ref state shards_.push_back(v->top_block_id()); } - auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader = new_cell_db_reader, + auto write_data = [shard = state->get_shard(), hash = state->root_cell()->get_hash(), cell_db_reader, + previous_state_cache = previous_state_cache_, + fast_serializer_enabled = opts_->get_fast_state_serializer_enabled(), cancellation_token = cancellation_token_source_.get_cancellation_token()](td::FileFd& fd) mutable { - auto res = vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31, std::move(cancellation_token)); - cell_db_reader->print_stats(); + if (fast_serializer_enabled) { + previous_state_cache->prepare_cache(shard); + } + auto new_cell_db_reader = std::make_shared(cell_db_reader, previous_state_cache->cache); + auto res = vm::std_boc_serialize_to_file_large(new_cell_db_reader, hash, fd, 31, std::move(cancellation_token)); + new_cell_db_reader->print_stats(); return res; }; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this)](td::Result R) { @@ -375,17 +378,21 @@ void AsyncStateSerializer::got_shard_handle(BlockHandle handle) { void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Ref state, std::shared_ptr cell_db_reader) { - if (!opts_->get_state_serializer_enabled()) { + if (!opts_->get_state_serializer_enabled() || auto_disabled_) { success_handler(); return; } LOG(ERROR) << "serializing shard state " << handle->id().id.to_str(); - prepare_previous_state_cache(state->get_shard()); - auto new_cell_db_reader = std::make_shared(cell_db_reader, previous_state_cache_); - auto write_data = [hash = state->root_cell()->get_hash(), cell_db_reader = new_cell_db_reader, + auto write_data = [shard = state->get_shard(), hash = state->root_cell()->get_hash(), cell_db_reader, + previous_state_cache = previous_state_cache_, + fast_serializer_enabled = opts_->get_fast_state_serializer_enabled(), cancellation_token = cancellation_token_source_.get_cancellation_token()](td::FileFd& fd) mutable { - auto res = vm::std_boc_serialize_to_file_large(cell_db_reader, hash, fd, 31, std::move(cancellation_token)); - cell_db_reader->print_stats(); + if (fast_serializer_enabled) { + previous_state_cache->prepare_cache(shard); + } + auto new_cell_db_reader = std::make_shared(cell_db_reader, previous_state_cache->cache); + auto res = vm::std_boc_serialize_to_file_large(new_cell_db_reader, hash, fd, 31, std::move(cancellation_token)); + new_cell_db_reader->print_stats(); return res; }; auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), handle](td::Result R) { @@ -427,6 +434,13 @@ void AsyncStateSerializer::update_options(td::Ref opts) } } +void AsyncStateSerializer::auto_disable_serializer(bool disabled) { + auto_disabled_ = disabled; + if (auto_disabled_) { + cancellation_token_source_.cancel(); + } +} + bool AsyncStateSerializer::need_monitor(ShardIdFull shard) { return opts_->need_monitor(shard); diff --git a/validator/state-serializer.hpp b/validator/state-serializer.hpp index 6d966f930..68606d1ea 100644 --- a/validator/state-serializer.hpp +++ b/validator/state-serializer.hpp @@ -37,6 +37,7 @@ class AsyncStateSerializer : public td::actor::Actor { bool saved_to_db_ = true; td::Ref opts_; + bool auto_disabled_ = false; td::CancellationTokenSource cancellation_token_source_; UnixTime last_known_key_block_ts_ = 0; @@ -48,11 +49,14 @@ class AsyncStateSerializer : public td::actor::Actor { bool have_masterchain_state_ = false; std::vector shards_; - std::vector> previous_state_files_; - std::shared_ptr>> previous_state_cache_; - std::vector previous_state_cur_shards_; + struct PreviousStateCache { + std::vector> state_files; + std::shared_ptr>> cache; + std::vector cur_shards; - void prepare_previous_state_cache(ShardIdFull shard); + void prepare_cache(ShardIdFull shard); + }; + std::shared_ptr previous_state_cache_; public: AsyncStateSerializer(BlockIdExt block_id, td::Ref opts, @@ -105,6 +109,7 @@ class AsyncStateSerializer : public td::actor::Actor { void success_handler(); void update_options(td::Ref opts); + void auto_disable_serializer(bool disabled); }; } // namespace validator From e9bd4823824efea4b16248de41dbebe5c73987c8 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Tue, 27 Aug 2024 18:17:43 +0300 Subject: [PATCH 24/37] Increase moderate misbehavior threshold --- lite-client/lite-client.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index 02a5fab67..da2fd6ff9 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -3737,10 +3737,10 @@ void TestNode::continue_check_validator_load3(std::unique_ptr Date: Wed, 28 Aug 2024 14:09:22 +0300 Subject: [PATCH 25/37] Allow unlimited catchain.getBlock requests (#1132) --- catchain/catchain-receiver-source.h | 2 +- catchain/catchain-receiver.cpp | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/catchain/catchain-receiver-source.h b/catchain/catchain-receiver-source.h index 55035e779..136906a02 100644 --- a/catchain/catchain-receiver-source.h +++ b/catchain/catchain-receiver-source.h @@ -61,7 +61,7 @@ class CatChainReceiverSource { virtual td::BufferSlice fork_proof() const = 0; virtual bool fork_is_found() const = 0; - // One block can be sent to one node only a limited number of times to prevent DoS + // One block can be sent to one node in catchain.getDifference only a limited number of times to prevent DoS virtual bool allow_send_block(CatChainBlockHash hash) = 0; static td::Result> create(CatChainReceiver *chain, PublicKey pub_key, diff --git a/catchain/catchain-receiver.cpp b/catchain/catchain-receiver.cpp index 82779e3be..ade4726fb 100644 --- a/catchain/catchain-receiver.cpp +++ b/catchain/catchain-receiver.cpp @@ -697,12 +697,8 @@ void CatChainReceiverImpl::process_query(adnl::AdnlNodeIdShort src, ton_api::cat } else { CatChainReceiverSource *S = get_source_by_adnl_id(src); CHECK(S != nullptr); - if (S->allow_send_block(it->second->get_hash())) { - promise.set_value(serialize_tl_object(create_tl_object(it->second->export_tl()), - true, it->second->get_payload().as_slice())); - } else { - promise.set_error(td::Status::Error("block was requested too many times")); - } + promise.set_value(serialize_tl_object(create_tl_object(it->second->export_tl()), + true, it->second->get_payload().as_slice())); } } From 97c57c3386e54efc33836ce1095baa08cce9344a Mon Sep 17 00:00:00 2001 From: neodix42 Date: Wed, 28 Aug 2024 18:59:28 +0400 Subject: [PATCH 26/37] add gh action to create docker image with a specified branch. Branch name will be used as image's tag name. (#1133) Co-authored-by: neodiX --- .../workflows/docker-ubuntu-branch-image.yml | 48 +++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 .github/workflows/docker-ubuntu-branch-image.yml diff --git a/.github/workflows/docker-ubuntu-branch-image.yml b/.github/workflows/docker-ubuntu-branch-image.yml new file mode 100644 index 000000000..d749afa2e --- /dev/null +++ b/.github/workflows/docker-ubuntu-branch-image.yml @@ -0,0 +1,48 @@ +name: Docker Ubuntu 22.04 branch image + +on: + workflow_dispatch: + push: + branches-ignore: + - master + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push: + runs-on: ubuntu-22.04 + steps: + - name: Check out repository + uses: actions/checkout@v3 + with: + submodules: 'recursive' + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.repository_owner }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Get tag as branch name + id: tag + run: | + echo "TAG=${GITHUB_REF##*/}" >> $GITHUB_OUTPUT + + - name: Build and push + id: docker_build + uses: docker/build-push-action@v6 + with: + platforms: linux/amd64 + push: true + context: ./ + tags: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ steps.tag.outputs.TAG }} From e08111159f7aa46219ed5bc198ac13a4caa8782d Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Fri, 30 Aug 2024 17:00:06 +0300 Subject: [PATCH 27/37] Dynamic catchain delays, state serialization improvements (#1140) * Validator improvements * Fix cancelling state serialization * Disable state serializer on all mainnet validators * Flag --catchain-max-block-delay-slow * Set default catchain-max-block-delay to 0.4, delay-slow to 1.0 --------- Co-authored-by: SpyCheese --- catchain/catchain-receiver.cpp | 6 ++++++ validator-engine/validator-engine.cpp | 17 +++++++++++++++-- validator-engine/validator-engine.hpp | 5 ++++- validator-session/validator-session-state.h | 8 ++++++++ validator-session/validator-session.cpp | 18 +++++++++++++++--- validator-session/validator-session.h | 2 +- validator-session/validator-session.hpp | 8 ++++++-- validator/impl/shard.cpp | 2 ++ validator/impl/shard.hpp | 4 ++++ validator/interfaces/shard.h | 1 + validator/manager.cpp | 18 ++++++------------ validator/state-serializer.cpp | 2 +- validator/validator-group.cpp | 10 ++++++---- validator/validator-options.hpp | 8 +++++++- validator/validator.h | 2 ++ 15 files changed, 84 insertions(+), 27 deletions(-) diff --git a/catchain/catchain-receiver.cpp b/catchain/catchain-receiver.cpp index ade4726fb..a6ecf0611 100644 --- a/catchain/catchain-receiver.cpp +++ b/catchain/catchain-receiver.cpp @@ -368,6 +368,12 @@ void CatChainReceiverImpl::add_block(td::BufferSlice payload, std::vectorheight_ + 1; + auto max_block_height = get_max_block_height(opts_, sources_.size()); + if (height > max_block_height) { + VLOG(CATCHAIN_WARNING) << this << ": cannot create block: max height exceeded (" << max_block_height << ")"; + active_send_ = false; + return; + } auto block_data = create_tl_object(std::move(prev), std::move(deps_arr)); auto block = create_tl_object(incarnation_, local_idx_, height, std::move(block_data), td::BufferSlice()); diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index a9a3b21d4..11f1a9a28 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -1448,6 +1448,9 @@ td::Status ValidatorEngine::load_global_config() { if (catchain_max_block_delay_) { validator_options_.write().set_catchain_max_block_delay(catchain_max_block_delay_.value()); } + if (catchain_max_block_delay_slow_) { + validator_options_.write().set_catchain_max_block_delay_slow(catchain_max_block_delay_slow_.value()); + } std::vector h; for (auto &x : conf.validator_->hardforks_) { @@ -4072,7 +4075,7 @@ int main(int argc, char *argv[]) { logger_ = td::TsFileLog::create(fname.str()).move_as_ok(); td::log_interface = logger_.get(); }); - p.add_checked_option('s', "state-ttl", "state will be gc'd after this time (in seconds) default=3600", + p.add_checked_option('s', "state-ttl", "state will be gc'd after this time (in seconds) default=86400", [&](td::Slice fname) { auto v = td::to_double(fname); if (v <= 0) { @@ -4233,7 +4236,7 @@ int main(int argc, char *argv[]) { "preload all cells from CellDb on startup (recommended to use with big enough celldb-cache-size and celldb-direct-io)", [&]() { acts.push_back([&x]() { td::actor::send_closure(x, &ValidatorEngine::set_celldb_preload_all, true); }); }); p.add_checked_option( - '\0', "catchain-max-block-delay", "delay before creating a new catchain block, in seconds (default: 0.5)", + '\0', "catchain-max-block-delay", "delay before creating a new catchain block, in seconds (default: 0.4)", [&](td::Slice s) -> td::Status { auto v = td::to_double(s); if (v < 0) { @@ -4242,6 +4245,16 @@ int main(int argc, char *argv[]) { acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_catchain_max_block_delay, v); }); return td::Status::OK(); }); + p.add_checked_option( + '\0', "catchain-max-block-delay-slow", "max extended catchain block delay (for too long rounds), (default: 1.0)", + [&](td::Slice s) -> td::Status { + auto v = td::to_double(s); + if (v < 0) { + return td::Status::Error("catchain-max-block-delay-slow should be non-negative"); + } + acts.push_back([&x, v]() { td::actor::send_closure(x, &ValidatorEngine::set_catchain_max_block_delay_slow, v); }); + return td::Status::OK(); + }); p.add_option( '\0', "fast-state-serializer", "faster persistent state serializer, but requires more RAM (enabled automatically on machines with >= 90GB RAM)", diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index b00c97130..7212a0b50 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -214,7 +214,7 @@ class ValidatorEngine : public td::actor::Actor { td::optional celldb_cache_size_ = 1LL << 30; bool celldb_direct_io_ = false; bool celldb_preload_all_ = false; - td::optional catchain_max_block_delay_; + td::optional catchain_max_block_delay_, catchain_max_block_delay_slow_; bool read_config_ = false; bool started_keyring_ = false; bool started_ = false; @@ -300,6 +300,9 @@ class ValidatorEngine : public td::actor::Actor { void set_catchain_max_block_delay(double value) { catchain_max_block_delay_ = value; } + void set_catchain_max_block_delay_slow(double value) { + catchain_max_block_delay_slow_ = value; + } void set_fast_state_serializer_enabled(bool value) { fast_state_serializer_enabled_ = value; } diff --git a/validator-session/validator-session-state.h b/validator-session/validator-session-state.h index 35910535d..4efaf77ff 100644 --- a/validator-session/validator-session-state.h +++ b/validator-session/validator-session-state.h @@ -478,6 +478,14 @@ class ValidatorSessionState : public ValidatorSessionDescription::RootObject { auto get_ts(td::uint32 src_idx) const { return att_->at(src_idx); } + td::uint32 cur_attempt_in_round(const ValidatorSessionDescription& desc) const { + td::uint32 first_attempt = cur_round_->get_first_attempt(desc.get_self_idx()); + td::uint32 cur_attempt = desc.get_attempt_seqno(desc.get_ts()); + if (cur_attempt < first_attempt || first_attempt == 0) { + return 0; + } + return cur_attempt - first_attempt; + } const SentBlock* choose_block_to_sign(ValidatorSessionDescription& desc, td::uint32 src_idx, bool& found) const; const SentBlock* get_committed_block(ValidatorSessionDescription& desc, td::uint32 seqno) const; diff --git a/validator-session/validator-session.cpp b/validator-session/validator-session.cpp index be5443785..246f7e58a 100644 --- a/validator-session/validator-session.cpp +++ b/validator-session/validator-session.cpp @@ -813,13 +813,25 @@ void ValidatorSessionImpl::request_new_block(bool now) { } else { double lambda = 10.0 / description().get_total_nodes(); double x = -1 / lambda * log(td::Random::fast(1, 999) * 0.001); - if (x > catchain_max_block_delay_) { // default = 0.5 - x = catchain_max_block_delay_; - } + x = std::min(x, get_current_max_block_delay()); // default = 0.4 td::actor::send_closure(catchain_, &catchain::CatChain::need_new_block, td::Timestamp::in(x)); } } +double ValidatorSessionImpl::get_current_max_block_delay() const { + td::uint32 att = real_state_->cur_attempt_in_round(*description_); + td::uint32 att1 = description_->opts().max_round_attempts; + if (att <= att1) { + return catchain_max_block_delay_; + } + td::uint32 att2 = att1 + 4; + if (att >= att2) { + return catchain_max_block_delay_slow_; + } + return catchain_max_block_delay_ + + (catchain_max_block_delay_slow_ - catchain_max_block_delay_) * (double)(att - att1) / (double)(att2 - att1); +} + void ValidatorSessionImpl::on_new_round(td::uint32 round) { if (round != 0) { CHECK(cur_round_ < round); diff --git a/validator-session/validator-session.h b/validator-session/validator-session.h index 2e1ed9b13..e60330b09 100644 --- a/validator-session/validator-session.h +++ b/validator-session/validator-session.h @@ -109,7 +109,7 @@ class ValidatorSession : public td::actor::Actor { virtual void get_validator_group_info_for_litequery( td::uint32 cur_round, td::Promise>> promise) = 0; - virtual void set_catchain_max_block_delay(double value) = 0; + virtual void set_catchain_max_block_delay(double delay, double delay_slow) = 0; static td::actor::ActorOwn create( catchain::CatChainSessionId session_id, ValidatorSessionOptions opts, PublicKeyHash local_id, diff --git a/validator-session/validator-session.hpp b/validator-session/validator-session.hpp index 2ee4885b9..39f196d8b 100644 --- a/validator-session/validator-session.hpp +++ b/validator-session/validator-session.hpp @@ -91,6 +91,7 @@ class ValidatorSessionImpl : public ValidatorSession { std::unique_ptr description_; double catchain_max_block_delay_ = 0.4; + double catchain_max_block_delay_slow_ = 1.0; void on_new_round(td::uint32 round); void on_catchain_started(); @@ -150,6 +151,7 @@ class ValidatorSessionImpl : public ValidatorSession { } void request_new_block(bool now); + double get_current_max_block_delay() const; void get_broadcast_p2p(PublicKeyHash node, ValidatorSessionFileHash file_hash, ValidatorSessionCollatedDataFileHash collated_data_file_hash, PublicKeyHash src, td::uint32 round, ValidatorSessionRootHash root_hash, td::Promise promise, @@ -191,8 +193,10 @@ class ValidatorSessionImpl : public ValidatorSession { void get_validator_group_info_for_litequery( td::uint32 cur_round, td::Promise>> promise) override; - void set_catchain_max_block_delay(double value) override { - catchain_max_block_delay_ = value; + + void set_catchain_max_block_delay(double delay, double delay_slow) override { + catchain_max_block_delay_ = delay; + catchain_max_block_delay_slow_ = delay_slow; } void process_blocks(std::vector blocks); diff --git a/validator/impl/shard.cpp b/validator/impl/shard.cpp index e899926a0..9c4245b6a 100644 --- a/validator/impl/shard.cpp +++ b/validator/impl/shard.cpp @@ -44,6 +44,7 @@ ShardStateQ::ShardStateQ(const ShardStateQ& other) , root(other.root) , lt(other.lt) , utime(other.utime) + , global_id_(other.global_id_) , before_split_(other.before_split_) , fake_split_(other.fake_split_) , fake_merge_(other.fake_merge_) { @@ -121,6 +122,7 @@ td::Status ShardStateQ::init() { } lt = info.gen_lt; utime = info.gen_utime; + global_id_ = info.global_id; before_split_ = info.before_split; block::ShardId id{info.shard_id}; ton::BlockId hdr_id{ton::ShardIdFull(id), info.seq_no}; diff --git a/validator/impl/shard.hpp b/validator/impl/shard.hpp index 99a9e8b08..d9a7dd655 100644 --- a/validator/impl/shard.hpp +++ b/validator/impl/shard.hpp @@ -38,6 +38,7 @@ class ShardStateQ : virtual public ShardState { Ref root; LogicalTime lt{0}; UnixTime utime{0}; + td::int32 global_id_{0}; bool before_split_{false}; bool fake_split_{false}; bool fake_merge_{false}; @@ -81,6 +82,9 @@ class ShardStateQ : virtual public ShardState { LogicalTime get_logical_time() const override { return lt; } + td::int32 get_global_id() const override { + return global_id_; + } td::optional get_master_ref() const override { return master_ref; } diff --git a/validator/interfaces/shard.h b/validator/interfaces/shard.h index 35fe4bc9a..85022e6dd 100644 --- a/validator/interfaces/shard.h +++ b/validator/interfaces/shard.h @@ -39,6 +39,7 @@ class ShardState : public td::CntObject { virtual UnixTime get_unix_time() const = 0; virtual LogicalTime get_logical_time() const = 0; + virtual td::int32 get_global_id() const = 0; virtual ShardIdFull get_shard() const = 0; virtual BlockSeqno get_seqno() const = 0; virtual BlockIdExt get_block_id() const = 0; diff --git a/validator/manager.cpp b/validator/manager.cpp index 8b7d0eb1d..79927d0ef 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2057,7 +2057,6 @@ void ValidatorManagerImpl::update_shards() { } } - bool validating_masterchain = false; if (allow_validate_) { for (auto &desc : new_shards) { auto shard = desc.first; @@ -2074,9 +2073,6 @@ void ValidatorManagerImpl::update_shards() { auto validator_id = get_validator(shard, val_set); if (!validator_id.is_zero()) { - if (shard.is_masterchain()) { - validating_masterchain = true; - } auto val_group_id = get_validator_set_id(shard, val_set, opts_hash, key_seqno, opts); if (force_recover) { @@ -2171,16 +2167,14 @@ void ValidatorManagerImpl::update_shards() { td::actor::send_closure(SelfId, &ValidatorManagerImpl::written_destroyed_validator_sessions, std::move(gc)); }); td::actor::send_closure(db_, &Db::update_destroyed_validator_sessions, gc_list_, std::move(P)); + } - if (!serializer_.empty()) { - td::actor::send_closure( - serializer_, &AsyncStateSerializer::auto_disable_serializer, - validating_masterchain && - last_masterchain_state_->get_validator_set(ShardIdFull{masterchainId})->export_vector().size() * 2 <= - last_masterchain_state_->get_total_validator_set(0)->export_vector().size()); - } + if (!serializer_.empty()) { + td::actor::send_closure( + serializer_, &AsyncStateSerializer::auto_disable_serializer, + !validator_groups_.empty() && last_masterchain_state_->get_global_id() == -239); // mainnet only } -} // namespace validator +} void ValidatorManagerImpl::written_destroyed_validator_sessions(std::vector> list) { for (auto &v : list) { diff --git a/validator/state-serializer.cpp b/validator/state-serializer.cpp index ef79d33cb..516d8177f 100644 --- a/validator/state-serializer.cpp +++ b/validator/state-serializer.cpp @@ -378,6 +378,7 @@ void AsyncStateSerializer::got_shard_handle(BlockHandle handle) { void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Ref state, std::shared_ptr cell_db_reader) { + next_idx_++; if (!opts_->get_state_serializer_enabled() || auto_disabled_) { success_handler(); return; @@ -406,7 +407,6 @@ void AsyncStateSerializer::got_shard_state(BlockHandle handle, td::Refid(), masterchain_handle_->id(), write_data, std::move(P)); - next_idx_++; } void AsyncStateSerializer::fail_handler(td::Status reason) { diff --git a/validator/validator-group.cpp b/validator/validator-group.cpp index 4b61c07cd..bd545f607 100644 --- a/validator/validator-group.cpp +++ b/validator/validator-group.cpp @@ -348,10 +348,12 @@ void ValidatorGroup::create_session() { << ".", allow_unsafe_self_blocks_resync_); } - if (opts_->get_catchain_max_block_delay()) { - td::actor::send_closure(session_, &validatorsession::ValidatorSession::set_catchain_max_block_delay, - opts_->get_catchain_max_block_delay().value()); - } + double catchain_delay = opts_->get_catchain_max_block_delay() ? opts_->get_catchain_max_block_delay().value() : 0.4; + double catchain_delay_slow = + std::max(catchain_delay, + opts_->get_catchain_max_block_delay_slow() ? opts_->get_catchain_max_block_delay_slow().value() : 1.0); + td::actor::send_closure(session_, &validatorsession::ValidatorSession::set_catchain_max_block_delay, catchain_delay, + catchain_delay_slow); if (started_) { td::actor::send_closure(session_, &validatorsession::ValidatorSession::start); } diff --git a/validator/validator-options.hpp b/validator/validator-options.hpp index 04aeb69bb..900a682fd 100644 --- a/validator/validator-options.hpp +++ b/validator/validator-options.hpp @@ -141,6 +141,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { td::optional get_catchain_max_block_delay() const override { return catchain_max_block_delay_; } + td::optional get_catchain_max_block_delay_slow() const override { + return catchain_max_block_delay_slow_; + } bool get_state_serializer_enabled() const override { return state_serializer_enabled_; } @@ -230,6 +233,9 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { void set_catchain_max_block_delay(double value) override { catchain_max_block_delay_ = value; } + void set_catchain_max_block_delay_slow(double value) override { + catchain_max_block_delay_slow_ = value; + } void set_state_serializer_enabled(bool value) override { state_serializer_enabled_ = value; } @@ -289,7 +295,7 @@ struct ValidatorManagerOptionsImpl : public ValidatorManagerOptions { td::optional celldb_cache_size_; bool celldb_direct_io_ = false; bool celldb_preload_all_ = false; - td::optional catchain_max_block_delay_; + td::optional catchain_max_block_delay_, catchain_max_block_delay_slow_; bool state_serializer_enabled_ = true; td::Ref collator_options_{true}; bool fast_state_serializer_enabled_ = false; diff --git a/validator/validator.h b/validator/validator.h index bbe22bb43..2171954a3 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -105,6 +105,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual bool get_celldb_direct_io() const = 0; virtual bool get_celldb_preload_all() const = 0; virtual td::optional get_catchain_max_block_delay() const = 0; + virtual td::optional get_catchain_max_block_delay_slow() const = 0; virtual bool get_state_serializer_enabled() const = 0; virtual td::Ref get_collator_options() const = 0; virtual bool get_fast_state_serializer_enabled() const = 0; @@ -136,6 +137,7 @@ struct ValidatorManagerOptions : public td::CntObject { virtual void set_celldb_direct_io(bool value) = 0; virtual void set_celldb_preload_all(bool value) = 0; virtual void set_catchain_max_block_delay(double value) = 0; + virtual void set_catchain_max_block_delay_slow(double value) = 0; virtual void set_state_serializer_enabled(bool value) = 0; virtual void set_collator_options(td::Ref value) = 0; virtual void set_fast_state_serializer_enabled(bool value) = 0; From b2b79fead1e90708b514d6d8dfd0dc63f1a4fe92 Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Tue, 3 Sep 2024 13:34:31 +0300 Subject: [PATCH 28/37] Ratelimit nochannel ADNL packets (#1147) * Get ADNL stats in validator console * Add timestamp to stats * Limit nochannel adnl packets --------- Co-authored-by: SpyCheese --- adnl/adnl-channel.cpp | 20 +- adnl/adnl-local-id.cpp | 97 +++- adnl/adnl-local-id.h | 19 + adnl/adnl-peer-table.cpp | 86 +++- adnl/adnl-peer-table.h | 2 +- adnl/adnl-peer-table.hpp | 4 +- adnl/adnl-peer.cpp | 436 ++++++++++++------ adnl/adnl-peer.h | 8 +- adnl/adnl-peer.hpp | 41 +- adnl/adnl.h | 2 + adnl/utils.hpp | 34 ++ tl/generate/scheme/ton_api.tl | 23 +- tl/generate/scheme/ton_api.tlo | Bin 92472 -> 94732 bytes .../validator-engine-console-query.cpp | 163 +++++++ .../validator-engine-console-query.h | 44 ++ .../validator-engine-console.cpp | 2 + validator-engine/validator-engine.cpp | 22 + validator-engine/validator-engine.hpp | 2 + 18 files changed, 834 insertions(+), 171 deletions(-) diff --git a/adnl/adnl-channel.cpp b/adnl/adnl-channel.cpp index 5c8229ca4..4da9d2eed 100644 --- a/adnl/adnl-channel.cpp +++ b/adnl/adnl-channel.cpp @@ -112,16 +112,16 @@ void AdnlChannelImpl::send_message(td::uint32 priority, td::actor::ActorId R) { - if (R.is_error()) { - VLOG(ADNL_WARNING) << id << ": dropping IN message: can not decrypt: " << R.move_as_error(); - } else { - auto packet = R.move_as_ok(); - packet.set_remote_addr(addr); - td::actor::send_closure(peer, &AdnlPeerPair::receive_packet_from_channel, channel_id, std::move(packet)); - } - }); + auto P = td::PromiseCreator::lambda([peer = peer_pair_, channel_id = channel_in_id_, addr, id = print_id(), + size = data.size()](td::Result R) { + if (R.is_error()) { + VLOG(ADNL_WARNING) << id << ": dropping IN message: can not decrypt: " << R.move_as_error(); + } else { + auto packet = R.move_as_ok(); + packet.set_remote_addr(addr); + td::actor::send_closure(peer, &AdnlPeerPair::receive_packet_from_channel, channel_id, std::move(packet), size); + } + }); decrypt(std::move(data), std::move(P)); } diff --git a/adnl/adnl-local-id.cpp b/adnl/adnl-local-id.cpp index b48182763..d72fc7bcc 100644 --- a/adnl/adnl-local-id.cpp +++ b/adnl/adnl-local-id.cpp @@ -41,20 +41,34 @@ AdnlAddressList AdnlLocalId::get_addr_list() const { } void AdnlLocalId::receive(td::IPAddress addr, td::BufferSlice data) { - auto P = td::PromiseCreator::lambda( - [peer_table = peer_table_, dst = short_id_, addr, id = print_id()](td::Result R) { - if (R.is_error()) { - VLOG(ADNL_WARNING) << id << ": dropping IN message: cannot decrypt: " << R.move_as_error(); - } else { - auto packet = R.move_as_ok(); - packet.set_remote_addr(addr); - td::actor::send_closure(peer_table, &AdnlPeerTable::receive_decrypted_packet, dst, std::move(packet)); - } - }); - + InboundRateLimiter& rate_limiter = inbound_rate_limiter_[addr]; + if (!rate_limiter.rate_limiter.take()) { + VLOG(ADNL_NOTICE) << this << ": dropping IN message: rate limit exceeded"; + add_dropped_packet_stats(addr); + return; + } + ++rate_limiter.currently_decrypting_packets; + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), peer_table = peer_table_, dst = short_id_, addr, + id = print_id(), size = data.size()](td::Result R) { + td::actor::send_closure(SelfId, &AdnlLocalId::decrypt_packet_done, addr); + if (R.is_error()) { + VLOG(ADNL_WARNING) << id << ": dropping IN message: cannot decrypt: " << R.move_as_error(); + } else { + auto packet = R.move_as_ok(); + packet.set_remote_addr(addr); + td::actor::send_closure(peer_table, &AdnlPeerTable::receive_decrypted_packet, dst, std::move(packet), size); + } + }); decrypt(std::move(data), std::move(P)); } +void AdnlLocalId::decrypt_packet_done(td::IPAddress addr) { + auto it = inbound_rate_limiter_.find(addr); + CHECK(it != inbound_rate_limiter_.end()); + --it->second.currently_decrypting_packets; + add_decrypted_packet_stats(addr); +} + void AdnlLocalId::deliver(AdnlNodeIdShort src, td::BufferSlice data) { auto s = std::move(data); for (auto &cb : cb_) { @@ -292,6 +306,67 @@ void AdnlLocalId::update_packet(AdnlPacket packet, bool update_id, bool sign, td } } +void AdnlLocalId::get_stats(td::Promise> promise) { + auto stats = create_tl_object(); + stats->short_id_ = short_id_.bits256_value(); + for (auto &[ip, x] : inbound_rate_limiter_) { + if (x.currently_decrypting_packets != 0) { + stats->current_decrypt_.push_back(create_tl_object( + ip.is_valid() ? PSTRING() << ip.get_ip_str() << ":" << ip.get_port() : "", x.currently_decrypting_packets)); + } + } + prepare_packet_stats(); + stats->packets_recent_ = packet_stats_prev_.tl(); + stats->packets_total_ = packet_stats_total_.tl(); + stats->packets_total_->ts_start_ = (double)Adnl::adnl_start_time(); + stats->packets_total_->ts_end_ = td::Clocks::system(); + promise.set_result(std::move(stats)); +} + +void AdnlLocalId::add_decrypted_packet_stats(td::IPAddress addr) { + prepare_packet_stats(); + ++packet_stats_cur_.decrypted_packets[addr]; + ++packet_stats_total_.decrypted_packets[addr]; +} + +void AdnlLocalId::add_dropped_packet_stats(td::IPAddress addr) { + prepare_packet_stats(); + ++packet_stats_cur_.dropped_packets[addr]; + ++packet_stats_total_.dropped_packets[addr]; +} + +void AdnlLocalId::prepare_packet_stats() { + double now = td::Clocks::system(); + if (now >= packet_stats_cur_.ts_end) { + packet_stats_prev_ = std::move(packet_stats_cur_); + packet_stats_cur_ = {}; + auto now_int = (int)td::Clocks::system(); + packet_stats_cur_.ts_start = (double)(now_int / 60 * 60); + packet_stats_cur_.ts_end = packet_stats_cur_.ts_start + 60.0; + if (packet_stats_prev_.ts_end < now - 60.0) { + packet_stats_prev_ = {}; + packet_stats_prev_.ts_end = packet_stats_cur_.ts_start; + packet_stats_prev_.ts_start = packet_stats_prev_.ts_end - 60.0; + } + } +} + +tl_object_ptr AdnlLocalId::PacketStats::tl() const { + auto obj = create_tl_object(); + obj->ts_start_ = ts_start; + obj->ts_end_ = ts_end; + for (const auto &[ip, packets] : decrypted_packets) { + obj->decrypted_packets_.push_back(create_tl_object( + ip.is_valid() ? PSTRING() << ip.get_ip_str() << ":" << ip.get_port() : "", packets)); + } + for (const auto &[ip, packets] : dropped_packets) { + obj->dropped_packets_.push_back(create_tl_object( + ip.is_valid() ? PSTRING() << ip.get_ip_str() << ":" << ip.get_port() : "", packets)); + } + return obj; +} + + } // namespace adnl } // namespace ton diff --git a/adnl/adnl-local-id.h b/adnl/adnl-local-id.h index c9ecfff16..be9d79d2c 100644 --- a/adnl/adnl-local-id.h +++ b/adnl/adnl-local-id.h @@ -55,6 +55,7 @@ class AdnlLocalId : public td::actor::Actor { void deliver(AdnlNodeIdShort src, td::BufferSlice data); void deliver_query(AdnlNodeIdShort src, td::BufferSlice data, td::Promise promise); void receive(td::IPAddress addr, td::BufferSlice data); + void decrypt_packet_done(td::IPAddress addr); void subscribe(std::string prefix, std::unique_ptr callback); void unsubscribe(std::string prefix); @@ -77,6 +78,8 @@ class AdnlLocalId : public td::actor::Actor { void update_packet(AdnlPacket packet, bool update_id, bool sign, td::int32 update_addr_list_if, td::int32 update_priority_addr_list_if, td::Promise promise); + void get_stats(td::Promise> promise); + td::uint32 get_mode() { return mode_; } @@ -101,6 +104,22 @@ class AdnlLocalId : public td::actor::Actor { td::uint32 mode_; + struct InboundRateLimiter { + RateLimiter rate_limiter = RateLimiter(75, 0.33); + td::uint64 currently_decrypting_packets = 0; + }; + std::map inbound_rate_limiter_; + struct PacketStats { + double ts_start = 0.0, ts_end = 0.0; + std::map decrypted_packets; + std::map dropped_packets; + + tl_object_ptr tl() const; + } packet_stats_cur_, packet_stats_prev_, packet_stats_total_; + void add_decrypted_packet_stats(td::IPAddress addr); + void add_dropped_packet_stats(td::IPAddress addr); + void prepare_packet_stats(); + void publish_address_list(); }; diff --git a/adnl/adnl-peer-table.cpp b/adnl/adnl-peer-table.cpp index 548915159..d885623a9 100644 --- a/adnl/adnl-peer-table.cpp +++ b/adnl/adnl-peer-table.cpp @@ -84,7 +84,7 @@ void AdnlPeerTableImpl::receive_packet(td::IPAddress addr, AdnlCategoryMask cat_ << " (len=" << (data.size() + 32) << ")"; } -void AdnlPeerTableImpl::receive_decrypted_packet(AdnlNodeIdShort dst, AdnlPacket packet) { +void AdnlPeerTableImpl::receive_decrypted_packet(AdnlNodeIdShort dst, AdnlPacket packet, td::uint64 serialized_size) { packet.run_basic_checks().ensure(); if (!packet.inited_from_short()) { @@ -119,7 +119,7 @@ void AdnlPeerTableImpl::receive_decrypted_packet(AdnlNodeIdShort dst, AdnlPacket return; } td::actor::send_closure(it->second, &AdnlPeer::receive_packet, dst, it2->second.mode, it2->second.local_id.get(), - std::move(packet)); + std::move(packet), serialized_size); } void AdnlPeerTableImpl::add_peer(AdnlNodeIdShort local_id, AdnlNodeIdFull id, AdnlAddressList addr_list) { @@ -385,6 +385,88 @@ void AdnlPeerTableImpl::get_conn_ip_str(AdnlNodeIdShort l_id, AdnlNodeIdShort p_ td::actor::send_closure(it->second, &AdnlPeer::get_conn_ip_str, l_id, std::move(promise)); } +void AdnlPeerTableImpl::get_stats(td::Promise> promise) { + class Cb : public td::actor::Actor { + public: + explicit Cb(td::Promise> promise) : promise_(std::move(promise)) { + } + + void got_local_id_stats(tl_object_ptr local_id) { + auto &local_id_stats = local_id_stats_[local_id->short_id_]; + if (local_id_stats) { + local_id->peers_ = std::move(local_id_stats->peers_); + } + local_id_stats = std::move(local_id); + dec_pending(); + } + + void got_peer_stats(std::vector> peer_pairs) { + for (auto &peer_pair : peer_pairs) { + auto &local_id_stats = local_id_stats_[peer_pair->local_id_]; + if (local_id_stats == nullptr) { + local_id_stats = create_tl_object(); + local_id_stats->short_id_ = peer_pair->local_id_; + } + local_id_stats->peers_.push_back(std::move(peer_pair)); + } + dec_pending(); + } + + void inc_pending() { + ++pending_; + } + + void dec_pending() { + CHECK(pending_ > 0); + --pending_; + if (pending_ == 0) { + auto stats = create_tl_object(); + stats->timestamp_ = td::Clocks::system(); + for (auto &[id, local_id_stats] : local_id_stats_) { + stats->local_ids_.push_back(std::move(local_id_stats)); + } + promise_.set_result(std::move(stats)); + stop(); + } + } + + private: + td::Promise> promise_; + size_t pending_ = 1; + + std::map> local_id_stats_; + }; + auto callback = td::actor::create_actor("adnlstats", std::move(promise)).release(); + + for (auto &[id, local_id] : local_ids_) { + td::actor::send_closure(callback, &Cb::inc_pending); + td::actor::send_closure(local_id.local_id, &AdnlLocalId::get_stats, + [id = id, callback](td::Result> R) { + if (R.is_error()) { + VLOG(ADNL_NOTICE) + << "failed to get stats for local id " << id << " : " << R.move_as_error(); + td::actor::send_closure(callback, &Cb::dec_pending); + } else { + td::actor::send_closure(callback, &Cb::got_local_id_stats, R.move_as_ok()); + } + }); + } + for (auto &[id, peer] : peers_) { + td::actor::send_closure(callback, &Cb::inc_pending); + td::actor::send_closure( + peer, &AdnlPeer::get_stats, + [id = id, callback](td::Result>> R) { + if (R.is_error()) { + VLOG(ADNL_NOTICE) << "failed to get stats for peer " << id << " : " << R.move_as_error(); + td::actor::send_closure(callback, &Cb::dec_pending); + } else { + td::actor::send_closure(callback, &Cb::got_peer_stats, R.move_as_ok()); + } + }); + } + td::actor::send_closure(callback, &Cb::dec_pending); +} + } // namespace adnl } // namespace ton diff --git a/adnl/adnl-peer-table.h b/adnl/adnl-peer-table.h index cb7da6135..055f32ac1 100644 --- a/adnl/adnl-peer-table.h +++ b/adnl/adnl-peer-table.h @@ -90,7 +90,7 @@ class AdnlPeerTable : public Adnl { virtual void answer_query(AdnlNodeIdShort src, AdnlNodeIdShort dst, AdnlQueryId query_id, td::BufferSlice data) = 0; virtual void receive_packet(td::IPAddress addr, AdnlCategoryMask cat_mask, td::BufferSlice data) = 0; - virtual void receive_decrypted_packet(AdnlNodeIdShort dst, AdnlPacket packet) = 0; + virtual void receive_decrypted_packet(AdnlNodeIdShort dst, AdnlPacket packet, td::uint64 serialized_size) = 0; virtual void send_message_in(AdnlNodeIdShort src, AdnlNodeIdShort dst, AdnlMessage message, td::uint32 flags) = 0; virtual void register_channel(AdnlChannelIdShort id, AdnlNodeIdShort local_id, diff --git a/adnl/adnl-peer-table.hpp b/adnl/adnl-peer-table.hpp index 1c30b84c7..12f64fcb2 100644 --- a/adnl/adnl-peer-table.hpp +++ b/adnl/adnl-peer-table.hpp @@ -44,7 +44,7 @@ class AdnlPeerTableImpl : public AdnlPeerTable { void add_static_nodes_from_config(AdnlNodesList nodes) override; void receive_packet(td::IPAddress addr, AdnlCategoryMask cat_mask, td::BufferSlice data) override; - void receive_decrypted_packet(AdnlNodeIdShort dst, AdnlPacket data) override; + void receive_decrypted_packet(AdnlNodeIdShort dst, AdnlPacket data, td::uint64 serialized_size) override; void send_message_in(AdnlNodeIdShort src, AdnlNodeIdShort dst, AdnlMessage message, td::uint32 flags) override; void send_message(AdnlNodeIdShort src, AdnlNodeIdShort dst, td::BufferSlice data) override { send_message_ex(src, dst, std::move(data), 0); @@ -108,6 +108,8 @@ class AdnlPeerTableImpl : public AdnlPeerTable { td::Promise, AdnlAddress>> promise) override; void get_conn_ip_str(AdnlNodeIdShort l_id, AdnlNodeIdShort p_id, td::Promise promise) override; + void get_stats(td::Promise> promise) override; + struct PrintId {}; PrintId print_id() const { return PrintId{}; diff --git a/adnl/adnl-peer.cpp b/adnl/adnl-peer.cpp index d82486fed..2b0077a8e 100644 --- a/adnl/adnl-peer.cpp +++ b/adnl/adnl-peer.cpp @@ -26,6 +26,7 @@ #include "td/utils/base64.h" #include "td/utils/Random.h" #include "auto/tl/ton_api.h" +#include "td/utils/overloaded.h" namespace ton { @@ -50,9 +51,13 @@ void AdnlPeerPairImpl::start_up() { } void AdnlPeerPairImpl::alarm() { - if (next_dht_query_at_ && next_dht_query_at_.is_in_past()) { - next_dht_query_at_ = td::Timestamp::never(); - discover(); + if (!disable_dht_query_) { + disable_dht_query_ = true; + if (next_dht_query_at_ && next_dht_query_at_.is_in_past()) { + next_dht_query_at_ = td::Timestamp::never(); + discover(); + } + alarm_timestamp().relax(next_dht_query_at_); } if (next_db_update_at_ && next_db_update_at_.is_in_past()) { if (received_from_db_ && received_from_static_nodes_ && !peer_id_.empty()) { @@ -68,11 +73,8 @@ void AdnlPeerPairImpl::alarm() { } if (retry_send_at_ && retry_send_at_.is_in_past()) { retry_send_at_ = td::Timestamp::never(); - auto messages = std::move(pending_messages_); - pending_messages_.clear(); - send_messages_in(std::move(messages), false); + send_messages_from_queue(); } - alarm_timestamp().relax(next_dht_query_at_); alarm_timestamp().relax(next_db_update_at_); alarm_timestamp().relax(retry_send_at_); } @@ -207,18 +209,24 @@ void AdnlPeerPairImpl::receive_packet_checked(AdnlPacket packet) { } } -void AdnlPeerPairImpl::receive_packet_from_channel(AdnlChannelIdShort id, AdnlPacket packet) { +void AdnlPeerPairImpl::receive_packet_from_channel(AdnlChannelIdShort id, AdnlPacket packet, + td::uint64 serialized_size) { + add_packet_stats(serialized_size, /* in = */ true, /* channel = */ true); if (id != channel_in_id_) { VLOG(ADNL_NOTICE) << this << ": dropping IN message: outdated channel id" << id; return; } - if (channel_inited_) { + if (channel_inited_ && !channel_ready_) { channel_ready_ = true; + if (!out_messages_queue_.empty()) { + td::actor::send_closure(actor_id(this), &AdnlPeerPairImpl::send_messages_from_queue); + } } receive_packet_checked(std::move(packet)); } -void AdnlPeerPairImpl::receive_packet(AdnlPacket packet) { +void AdnlPeerPairImpl::receive_packet(AdnlPacket packet, td::uint64 serialized_size) { + add_packet_stats(serialized_size, /* in = */ true, /* channel = */ false); packet.run_basic_checks().ensure(); if (!encryptor_) { @@ -239,132 +247,132 @@ void AdnlPeerPairImpl::deliver_message(AdnlMessage message) { message.visit([&](const auto &obj) { this->process_message(obj); }); } -void AdnlPeerPairImpl::send_messages_in(std::vector messages, bool allow_postpone) { - for (td::int32 idx = 0; idx < 2; idx++) { - std::vector not_sent; +void AdnlPeerPairImpl::send_messages_from_queue() { + while (!out_messages_queue_.empty() && out_messages_queue_.front().second.is_in_past()) { + out_messages_queue_total_size_ -= out_messages_queue_.front().first.size(); + add_expired_msg_stats(out_messages_queue_.front().first.size()); + out_messages_queue_.pop(); + VLOG(ADNL_NOTICE) << this << ": dropping OUT message: message in queue expired"; + } + if (out_messages_queue_.empty()) { + return; + } - auto connR = get_conn(idx == 1); - if (connR.is_error()) { - if (!allow_postpone) { - VLOG(ADNL_NOTICE) << this << ": dropping OUT messages: cannot get conn: " << connR.move_as_error(); - return; - } - VLOG(ADNL_INFO) << this << ": delaying OUT messages: cannot get conn: " << connR.move_as_error(); - if (!retry_send_at_) { - retry_send_at_.relax(td::Timestamp::in(10.0)); - alarm_timestamp().relax(retry_send_at_); - } - for (auto &m : messages) { - pending_messages_.push_back(std::move(m)); - } + auto connR = get_conn(); + if (connR.is_error()) { + disable_dht_query_ = false; + retry_send_at_.relax(td::Timestamp::in(message_in_queue_ttl_ - 1.0)); + alarm_timestamp().relax(retry_send_at_); + VLOG(ADNL_INFO) << this << ": delaying OUT messages: cannot get conn: " << connR.move_as_error(); + return; + } + disable_dht_query_ = true; + auto C = connR.move_as_ok(); + auto conn = std::move(C.first); + bool is_direct = C.second; + + bool first = !skip_init_packet_; + while (!out_messages_queue_.empty()) { + bool try_reinit = try_reinit_at_ && try_reinit_at_.is_in_past(); + bool via_channel = channel_ready_ && !try_reinit; + if (!via_channel && !nochannel_rate_limiter_.take()) { + alarm_timestamp().relax(retry_send_at_ = nochannel_rate_limiter_.ready_at()); return; } - auto C = connR.move_as_ok(); - bool is_direct = C.second; - auto conn = std::move(C.first); - if (idx == 1) { - CHECK(is_direct); + if (try_reinit) { + try_reinit_at_ = td::Timestamp::in(td::Random::fast(0.5, 1.5)); } + respond_with_nop_after_ = td::Timestamp::in(td::Random::fast(1.0, 2.0)); - size_t ptr = 0; - bool first = true; - do { - respond_with_nop_after_ = td::Timestamp::in(td::Random::fast(1.0, 2.0)); - bool try_reinit = try_reinit_at_ && try_reinit_at_.is_in_past(); - if (try_reinit) { - try_reinit_at_ = td::Timestamp::in(td::Random::fast(0.5, 1.5)); - } - bool via_channel = channel_ready_ && !try_reinit; - size_t s = (via_channel ? channel_packet_header_max_size() : packet_header_max_size()); - if (first) { - s += 2 * addr_list_max_size(); - } - - AdnlPacket packet; - packet.set_seqno(++out_seqno_); - packet.set_confirm_seqno(in_seqno_); + size_t s = (via_channel ? channel_packet_header_max_size() : packet_header_max_size()); + if (first) { + s += 2 * addr_list_max_size(); + } - if (first) { - if (!channel_inited_) { - auto M = adnlmessage::AdnlMessageCreateChannel{channel_pub_, channel_pk_date_}; - s += M.size(); - packet.add_message(std::move(M)); - } else if (!channel_ready_) { - auto M = adnlmessage::AdnlMessageConfirmChannel{channel_pub_, peer_channel_pub_, channel_pk_date_}; - s += M.size(); - packet.add_message(std::move(M)); - } + AdnlPacket packet; + packet.set_seqno(++out_seqno_); + packet.set_confirm_seqno(in_seqno_); + + if (first) { + if (!channel_inited_) { + auto M = adnlmessage::AdnlMessageCreateChannel{channel_pub_, channel_pk_date_}; + s += M.size(); + packet.add_message(std::move(M)); + } else if (!channel_ready_) { + auto M = adnlmessage::AdnlMessageConfirmChannel{channel_pub_, peer_channel_pub_, channel_pk_date_}; + s += M.size(); + packet.add_message(std::move(M)); } + } - if (!addr_list_.empty()) { - packet.set_received_addr_list_version(addr_list_.version()); + if (!addr_list_.empty()) { + packet.set_received_addr_list_version(addr_list_.version()); + } + if (!priority_addr_list_.empty()) { + packet.set_received_priority_addr_list_version(priority_addr_list_.version()); + } + + skip_init_packet_ = true; + while (!out_messages_queue_.empty()) { + auto &M = out_messages_queue_.front().first; + if (!is_direct && (M.flags() & Adnl::SendFlags::direct_only)) { + out_messages_queue_total_size_ -= M.size(); + out_messages_queue_.pop(); + continue; } - if (!priority_addr_list_.empty()) { - packet.set_received_priority_addr_list_version(priority_addr_list_.version()); + CHECK(M.size() <= get_mtu()); + if (s + M.size() <= AdnlNetworkManager::get_mtu()) { + s += M.size(); + out_messages_queue_total_size_ -= M.size(); + packet.add_message(M.release()); + out_messages_queue_.pop(); + skip_init_packet_ = false; + } else { + break; } + } + + if (!via_channel) { + packet.set_reinit_date(Adnl::adnl_start_time(), reinit_date_); + packet.set_source(local_id_); + } - while (ptr < messages.size()) { - auto &M = messages[ptr]; - if (!is_direct && (M.flags() & Adnl::SendFlags::direct_only)) { - not_sent.push_back(std::move(M)); - continue; + if (!first) { + if (!channel_inited_) { + auto M = adnlmessage::AdnlMessageCreateChannel{channel_pub_, channel_pk_date_}; + if (s + M.size() <= AdnlNetworkManager::get_mtu()) { + s += M.size(); + packet.add_message(std::move(M)); } - CHECK(M.size() <= get_mtu()); + } else if (!channel_ready_) { + auto M = adnlmessage::AdnlMessageConfirmChannel{channel_pub_, peer_channel_pub_, channel_pk_date_}; if (s + M.size() <= AdnlNetworkManager::get_mtu()) { s += M.size(); - packet.add_message(M.release()); - ptr++; - } else { - break; + packet.add_message(std::move(M)); } } + } - if (!via_channel) { - packet.set_reinit_date(Adnl::adnl_start_time(), reinit_date_); - packet.set_source(local_id_); - } - - if (!first) { - if (!channel_inited_) { - auto M = adnlmessage::AdnlMessageCreateChannel{channel_pub_, channel_pk_date_}; - if (s + M.size() <= AdnlNetworkManager::get_mtu()) { - s += M.size(); - packet.add_message(std::move(M)); - } - } else if (!channel_ready_) { - auto M = adnlmessage::AdnlMessageConfirmChannel{channel_pub_, peer_channel_pub_, channel_pk_date_}; - if (s + M.size() <= AdnlNetworkManager::get_mtu()) { - s += M.size(); - packet.add_message(std::move(M)); - } - } + packet.run_basic_checks().ensure(); + auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), conn, id = print_id(), + via_channel](td::Result res) { + if (res.is_error()) { + LOG(ERROR) << id << ": dropping OUT message: error while creating packet: " << res.move_as_error(); + } else { + td::actor::send_closure(SelfId, &AdnlPeerPairImpl::send_packet_continue, res.move_as_ok(), conn, via_channel); } - - packet.run_basic_checks().ensure(); - auto P = td::PromiseCreator::lambda([SelfId = actor_id(this), conn, id = print_id(), - via_channel](td::Result res) { - if (res.is_error()) { - LOG(ERROR) << id << ": dropping OUT message: error while creating packet: " << res.move_as_error(); - } else { - td::actor::send_closure(SelfId, &AdnlPeerPairImpl::send_packet_continue, res.move_as_ok(), conn, via_channel); - } - }); - - td::actor::send_closure(local_actor_, &AdnlLocalId::update_packet, std::move(packet), - (!channel_ready_ && ack_seqno_ == 0 && in_seqno_ == 0) || try_reinit, !via_channel, - (first || s + addr_list_max_size() <= AdnlNetworkManager::get_mtu()) - ? (try_reinit ? 0 : peer_recv_addr_list_version_) - : 0x7fffffff, - (first || s + 2 * addr_list_max_size() <= AdnlNetworkManager::get_mtu()) - ? peer_recv_priority_addr_list_version_ - : 0x7fffffff, - std::move(P)); - first = false; - } while (ptr < messages.size()); - messages = std::move(not_sent); - if (!messages.size()) { - break; - } + }); + + td::actor::send_closure(local_actor_, &AdnlLocalId::update_packet, std::move(packet), + (!channel_ready_ && ack_seqno_ == 0 && in_seqno_ == 0) || try_reinit, !via_channel, + (first || s + addr_list_max_size() <= AdnlNetworkManager::get_mtu()) + ? (try_reinit ? 0 : peer_recv_addr_list_version_) + : 0x7fffffff, + (first || s + 2 * addr_list_max_size() <= AdnlNetworkManager::get_mtu()) + ? peer_recv_priority_addr_list_version_ + : 0x7fffffff, + std::move(P)); + first = false; } } @@ -395,7 +403,11 @@ void AdnlPeerPairImpl::send_messages(std::vector messages) } } } - send_messages_in(std::move(new_vec), true); + for (auto &m : new_vec) { + out_messages_queue_total_size_ += m.size(); + out_messages_queue_.emplace(std::move(m), td::Timestamp::in(message_in_queue_ttl_)); + } + send_messages_from_queue(); } void AdnlPeerPairImpl::send_packet_continue(AdnlPacket packet, td::actor::ActorId conn, @@ -407,6 +419,7 @@ void AdnlPeerPairImpl::send_packet_continue(AdnlPacket packet, td::actor::ActorI auto B = serialize_tl_object(packet.tl(), true); if (via_channel) { if (channel_ready_) { + add_packet_stats(B.size(), /* in = */ false, /* channel = */ true); td::actor::send_closure(channel_, &AdnlChannel::send_message, priority_, conn, std::move(B)); } else { VLOG(ADNL_WARNING) << this << ": dropping OUT message [" << local_id_ << "->" << peer_id_short_ @@ -434,6 +447,7 @@ void AdnlPeerPairImpl::send_packet_continue(AdnlPacket packet, td::actor::ActorI S.remove_prefix(32); S.copy_from(X.as_slice()); + add_packet_stats(B.size(), /* in = */ false, /* channel = */ false); td::actor::send_closure(conn, &AdnlNetworkConnection::send, local_id_, peer_id_short_, priority_, std::move(enc)); } @@ -520,7 +534,10 @@ void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageConfirmChan VLOG(ADNL_NOTICE) << this << ": received adnl.message.confirmChannel with old key"; return; } - channel_ready_ = true; + if (!channel_ready_) { + channel_ready_ = true; + send_messages_from_queue(); + } } void AdnlPeerPairImpl::process_message(const adnlmessage::AdnlMessageCustom &message) { @@ -674,7 +691,7 @@ void AdnlPeerPairImpl::reinit(td::int32 date) { } } -td::Result, bool>> AdnlPeerPairImpl::get_conn(bool direct_only) { +td::Result, bool>> AdnlPeerPairImpl::get_conn() { if (!priority_addr_list_.empty() && priority_addr_list_.expire_at() < td::Clocks::system()) { priority_addr_list_ = AdnlAddressList{}; priority_conns_.clear(); @@ -692,14 +709,18 @@ td::Result, bool>> AdnlPeerP } } - for (auto &conn : priority_conns_) { - if (conn.ready() && (!direct_only || conn.is_direct())) { - return std::make_pair(conn.conn.get(), conn.is_direct()); + for (int direct_only = 1; direct_only >= 0; --direct_only) { + for (auto &conn : priority_conns_) { + if (conn.ready() && (!direct_only || conn.is_direct())) { + return std::make_pair(conn.conn.get(), conn.is_direct()); + } } } - for (auto &conn : conns_) { - if (conn.ready() && (!direct_only || conn.is_direct())) { - return std::make_pair(conn.conn.get(), conn.is_direct()); + for (int direct_only = 1; direct_only >= 0; --direct_only) { + for (auto &conn : conns_) { + if (conn.ready() && (!direct_only || conn.is_direct())) { + return std::make_pair(conn.conn.get(), conn.is_direct()); + } } } return td::Status::Error(ErrorCode::notready, "no active connections"); @@ -787,6 +808,47 @@ void AdnlPeerPairImpl::get_conn_ip_str(td::Promise promise) { promise.set_value("undefined"); } +void AdnlPeerPairImpl::get_stats(td::Promise> promise) { + auto stats = create_tl_object(); + stats->local_id_ = local_id_.bits256_value(); + stats->peer_id_ = peer_id_short_.bits256_value(); + for (const AdnlAddress &addr : addr_list_.addrs()) { + ton_api::downcast_call(*addr->tl(), td::overloaded( + [&](const ton_api::adnl_address_udp &obj) { + stats->ip_str_ = PSTRING() << td::IPAddress::ipv4_to_str(obj.ip_) << ":" + << obj.port_; + }, + [&](const auto &) {})); + if (!stats->ip_str_.empty()) { + break; + } + } + + prepare_packet_stats(); + stats->last_in_packet_ts_ = last_in_packet_ts_; + stats->last_out_packet_ts_ = last_out_packet_ts_; + stats->packets_total_ = packet_stats_total_.tl(); + stats->packets_total_->ts_start_ = started_ts_; + stats->packets_total_->ts_end_ = td::Clocks::system(); + stats->packets_recent_ = packet_stats_prev_.tl(); + + if (channel_ready_) { + stats->channel_status_ = 2; + } else if (channel_inited_) { + stats->channel_status_ = 1; + } else { + stats->channel_status_ = 0; + } + stats->try_reinit_at_ = (try_reinit_at_ ? try_reinit_at_.at_unix() : 0.0); + stats->connection_ready_ = + std::any_of(conns_.begin(), conns_.end(), [](const Conn &conn) { return conn.ready(); }) || + std::any_of(priority_conns_.begin(), priority_conns_.end(), [](const Conn &conn) { return conn.ready(); }); + stats->out_queue_messages_ = out_messages_queue_.size(); + stats->out_queue_bytes_ = out_messages_queue_total_size_; + + promise.set_result(std::move(stats)); +} + void AdnlPeerImpl::update_id(AdnlNodeIdFull id) { CHECK(id.compute_short_id() == peer_id_short_); if (!peer_id_.empty()) { @@ -810,10 +872,8 @@ void AdnlPeerPairImpl::Conn::create_conn(td::actor::ActorId pe void AdnlPeerPairImpl::conn_change_state(AdnlConnectionIdShort id, bool ready) { if (ready) { - if (pending_messages_.size() > 0) { - auto messages = std::move(pending_messages_); - pending_messages_.clear(); - send_messages_in(std::move(messages), true); + if (out_messages_queue_.empty()) { + send_messages_from_queue(); } } } @@ -835,7 +895,7 @@ td::actor::ActorOwn AdnlPeer::create(td::actor::ActorId dst_actor, - AdnlPacket packet) { + AdnlPacket packet, td::uint64 serialized_size) { if (packet.inited_from()) { update_id(packet.from()); } @@ -853,7 +913,7 @@ void AdnlPeerImpl::receive_packet(AdnlNodeIdShort dst, td::uint32 dst_mode, td:: } } - td::actor::send_closure(it->second.get(), &AdnlPeerPair::receive_packet, std::move(packet)); + td::actor::send_closure(it->second.get(), &AdnlPeerPair::receive_packet, std::move(packet), serialized_size); } void AdnlPeerImpl::send_messages(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId src_actor, @@ -933,6 +993,56 @@ void AdnlPeerImpl::update_addr_list(AdnlNodeIdShort local_id, td::uint32 local_m td::actor::send_closure(it->second, &AdnlPeerPair::update_addr_list, std::move(addr_list)); } +void AdnlPeerImpl::get_stats(td::Promise>> promise) { + class Cb : public td::actor::Actor { + public: + explicit Cb(td::Promise>> promise) + : promise_(std::move(promise)) { + } + + void got_peer_pair_stats(tl_object_ptr peer_pair) { + result_.push_back(std::move(peer_pair)); + dec_pending(); + } + + void inc_pending() { + ++pending_; + } + + void dec_pending() { + CHECK(pending_ > 0); + --pending_; + if (pending_ == 0) { + promise_.set_result(std::move(result_)); + stop(); + } + } + + private: + td::Promise>> promise_; + size_t pending_ = 1; + std::vector> result_; + }; + auto callback = td::actor::create_actor("adnlpeerstats", std::move(promise)).release(); + + for (auto &[local_id, peer_pair] : peer_pairs_) { + td::actor::send_closure(callback, &Cb::inc_pending); + td::actor::send_closure(peer_pair, &AdnlPeerPair::get_stats, + [local_id = local_id, peer_id = peer_id_short_, + callback](td::Result> R) { + if (R.is_error()) { + VLOG(ADNL_NOTICE) << "failed to get stats for peer pair " << peer_id << "->" << local_id + << " : " << R.move_as_error(); + td::actor::send_closure(callback, &Cb::dec_pending); + } else { + td::actor::send_closure(callback, &Cb::got_peer_pair_stats, R.move_as_ok()); + } + }); + } + td::actor::send_closure(callback, &Cb::dec_pending); +} + + void AdnlPeerPairImpl::got_data_from_db(td::Result R) { received_from_db_ = false; if (R.is_error()) { @@ -1016,6 +1126,66 @@ void AdnlPeerPairImpl::request_reverse_ping_result(td::Result R) { } } +void AdnlPeerPairImpl::add_packet_stats(td::uint64 bytes, bool in, bool channel) { + prepare_packet_stats(); + auto add_stats = [&](PacketStats &stats) { + if (in) { + ++stats.in_packets; + stats.in_bytes += bytes; + if (channel) { + ++stats.in_packets_channel; + stats.in_bytes_channel += bytes; + } + } else { + ++stats.out_packets; + stats.out_bytes += bytes; + if (channel) { + ++stats.out_packets_channel; + stats.out_bytes_channel += bytes; + } + } + }; + add_stats(packet_stats_cur_); + add_stats(packet_stats_total_); + if (in) { + last_in_packet_ts_ = td::Clocks::system(); + } else { + last_out_packet_ts_ = td::Clocks::system(); + } +} + +void AdnlPeerPairImpl::add_expired_msg_stats(td::uint64 bytes) { + prepare_packet_stats(); + auto add_stats = [&](PacketStats &stats) { + ++stats.out_expired_messages; + stats.out_expired_bytes += bytes; + }; + add_stats(packet_stats_cur_); + add_stats(packet_stats_total_); +} + +void AdnlPeerPairImpl::prepare_packet_stats() { + double now = td::Clocks::system(); + if (now >= packet_stats_cur_.ts_end) { + packet_stats_prev_ = std::move(packet_stats_cur_); + packet_stats_cur_ = {}; + auto now_int = (int)now; + packet_stats_cur_.ts_start = (double)(now_int / 60 * 60); + packet_stats_cur_.ts_end = packet_stats_cur_.ts_start + 60.0; + if (packet_stats_prev_.ts_end < now - 60.0) { + packet_stats_prev_ = {}; + packet_stats_prev_.ts_end = packet_stats_cur_.ts_start; + packet_stats_prev_.ts_start = packet_stats_prev_.ts_end - 60.0; + } + } +} + +tl_object_ptr AdnlPeerPairImpl::PacketStats::tl() const { + return create_tl_object(ts_start, ts_end, in_packets, in_bytes, in_packets_channel, + in_bytes_channel, out_packets, out_bytes, out_packets_channel, + out_bytes_channel, out_expired_messages, out_expired_bytes); +} + } // namespace adnl } // namespace ton diff --git a/adnl/adnl-peer.h b/adnl/adnl-peer.h index 8488e82ee..b7d6adc0f 100644 --- a/adnl/adnl-peer.h +++ b/adnl/adnl-peer.h @@ -39,9 +39,9 @@ class AdnlPeer; class AdnlPeerPair : public td::actor::Actor { public: - virtual void receive_packet_from_channel(AdnlChannelIdShort id, AdnlPacket packet) = 0; + virtual void receive_packet_from_channel(AdnlChannelIdShort id, AdnlPacket packet, td::uint64 serialized_size) = 0; virtual void receive_packet_checked(AdnlPacket packet) = 0; - virtual void receive_packet(AdnlPacket packet) = 0; + virtual void receive_packet(AdnlPacket packet, td::uint64 serialized_size) = 0; virtual void send_messages(std::vector message) = 0; inline void send_message(OutboundAdnlMessage message) { @@ -59,6 +59,7 @@ class AdnlPeerPair : public td::actor::Actor { virtual void update_peer_id(AdnlNodeIdFull id) = 0; virtual void update_addr_list(AdnlAddressList addr_list) = 0; virtual void get_conn_ip_str(td::Promise promise) = 0; + virtual void get_stats(td::Promise> promise) = 0; static td::actor::ActorOwn create(td::actor::ActorId network_manager, td::actor::ActorId peer_table, td::uint32 local_mode, @@ -71,7 +72,7 @@ class AdnlPeerPair : public td::actor::Actor { class AdnlPeer : public td::actor::Actor { public: virtual void receive_packet(AdnlNodeIdShort dst, td::uint32 dst_mode, td::actor::ActorId dst_actor, - AdnlPacket message) = 0; + AdnlPacket message, td::uint64 serialized_size) = 0; virtual void send_messages(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId src_actor, std::vector messages) = 0; virtual void send_query(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId src_actor, @@ -100,6 +101,7 @@ class AdnlPeer : public td::actor::Actor { td::actor::ActorId local_actor, AdnlAddressList addr_list) = 0; virtual void update_dht_node(td::actor::ActorId dht_node) = 0; virtual void get_conn_ip_str(AdnlNodeIdShort l_id, td::Promise promise) = 0; + virtual void get_stats(td::Promise>> promise) = 0; }; } // namespace adnl diff --git a/adnl/adnl-peer.hpp b/adnl/adnl-peer.hpp index 40c9eb088..d25a24cf4 100644 --- a/adnl/adnl-peer.hpp +++ b/adnl/adnl-peer.hpp @@ -20,6 +20,7 @@ #include #include +#include #include "adnl-peer.h" #include "adnl-peer-table.h" @@ -66,12 +67,12 @@ class AdnlPeerPairImpl : public AdnlPeerPair { void discover(); - void receive_packet_from_channel(AdnlChannelIdShort id, AdnlPacket packet) override; + void receive_packet_from_channel(AdnlChannelIdShort id, AdnlPacket packet, td::uint64 serialized_size) override; void receive_packet_checked(AdnlPacket packet) override; - void receive_packet(AdnlPacket packet) override; + void receive_packet(AdnlPacket packet, td::uint64 serialized_size) override; void deliver_message(AdnlMessage message); - void send_messages_in(std::vector messages, bool allow_postpone); + void send_messages_from_queue(); void send_messages(std::vector messages) override; void send_packet_continue(AdnlPacket packet, td::actor::ActorId conn, bool via_channel); void send_query(std::string name, td::Promise promise, td::Timestamp timeout, td::BufferSlice data, @@ -89,6 +90,7 @@ class AdnlPeerPairImpl : public AdnlPeerPair { void update_peer_id(AdnlNodeIdFull id) override; void get_conn_ip_str(td::Promise promise) override; + void get_stats(td::Promise> promise) override; void got_data_from_db(td::Result R); void got_data_from_static_nodes(td::Result R); @@ -124,7 +126,7 @@ class AdnlPeerPairImpl : public AdnlPeerPair { private: void respond_with_nop(); void reinit(td::int32 date); - td::Result, bool>> get_conn(bool direct_only); + td::Result, bool>> get_conn(); void create_channel(pubkeys::Ed25519 pub, td::uint32 date); bool received_packet(td::uint64 seqno) const { @@ -183,11 +185,11 @@ class AdnlPeerPairImpl : public AdnlPeerPair { Conn() { } - bool ready() { + bool ready() const { return !conn.empty() && conn.get_actor_unsafe().is_active(); } - bool is_direct() { + bool is_direct() const { return addr->is_public(); } @@ -195,7 +197,14 @@ class AdnlPeerPairImpl : public AdnlPeerPair { td::actor::ActorId adnl); }; - std::vector pending_messages_; + // Messages waiting for connection or for nochannel rate limiter + std::queue> out_messages_queue_; + td::uint64 out_messages_queue_total_size_ = 0; + RateLimiter nochannel_rate_limiter_ = RateLimiter(50, 0.5); // max 50, period = 0.5s + td::Timestamp retry_send_at_ = td::Timestamp::never(); + bool disable_dht_query_ = false; + bool skip_init_packet_ = false; + double message_in_queue_ttl_ = 10.0; td::actor::ActorId network_manager_; td::actor::ActorId peer_table_; @@ -254,7 +263,6 @@ class AdnlPeerPairImpl : public AdnlPeerPair { td::Timestamp next_dht_query_at_ = td::Timestamp::never(); td::Timestamp next_db_update_at_ = td::Timestamp::never(); - td::Timestamp retry_send_at_ = td::Timestamp::never(); td::Timestamp last_received_packet_ = td::Timestamp::never(); td::Timestamp try_reinit_at_ = td::Timestamp::never(); @@ -262,12 +270,26 @@ class AdnlPeerPairImpl : public AdnlPeerPair { bool has_reverse_addr_ = false; td::Timestamp request_reverse_ping_after_ = td::Timestamp::now(); bool request_reverse_ping_active_ = false; + + struct PacketStats { + double ts_start = 0.0, ts_end = 0.0; + td::uint64 in_packets = 0, in_bytes = 0, in_packets_channel = 0, in_bytes_channel = 0; + td::uint64 out_packets = 0, out_bytes = 0, out_packets_channel = 0, out_bytes_channel = 0; + td::uint64 out_expired_messages = 0, out_expired_bytes = 0; + + tl_object_ptr tl() const; + } packet_stats_cur_, packet_stats_prev_, packet_stats_total_; + double last_in_packet_ts_ = 0.0, last_out_packet_ts_ = 0.0; + double started_ts_ = td::Clocks::system(); + void add_packet_stats(td::uint64 bytes, bool in, bool channel); + void add_expired_msg_stats(td::uint64 bytes); + void prepare_packet_stats(); }; class AdnlPeerImpl : public AdnlPeer { public: void receive_packet(AdnlNodeIdShort dst, td::uint32 dst_mode, td::actor::ActorId dst_actor, - AdnlPacket packet) override; + AdnlPacket packet, td::uint64 serialized_size) override; void send_messages(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId src_actor, std::vector messages) override; void send_query(AdnlNodeIdShort src, td::uint32 src_mode, td::actor::ActorId src_actor, std::string name, @@ -280,6 +302,7 @@ class AdnlPeerImpl : public AdnlPeer { AdnlAddressList addr_list) override; void update_dht_node(td::actor::ActorId dht_node) override; void get_conn_ip_str(AdnlNodeIdShort l_id, td::Promise promise) override; + void get_stats(td::Promise>> promise) override; //void check_signature(td::BufferSlice data, td::BufferSlice signature, td::Promise promise) override; AdnlPeerImpl(td::actor::ActorId network_manager, td::actor::ActorId peer_table, diff --git a/adnl/adnl.h b/adnl/adnl.h index a1c39d5e4..a276e0c21 100644 --- a/adnl/adnl.h +++ b/adnl/adnl.h @@ -121,6 +121,8 @@ class Adnl : public AdnlSenderInterface { virtual void create_tunnel(AdnlNodeIdShort dst, td::uint32 size, td::Promise, AdnlAddress>> promise) = 0; + virtual void get_stats(td::Promise> promise) = 0; + static td::actor::ActorOwn create(std::string db, td::actor::ActorId keyring); static std::string int_to_bytestring(td::int32 id) { diff --git a/adnl/utils.hpp b/adnl/utils.hpp index 50aec2efb..18d3f2074 100644 --- a/adnl/utils.hpp +++ b/adnl/utils.hpp @@ -40,6 +40,40 @@ inline bool adnl_node_is_older(AdnlNode &a, AdnlNode &b) { return a.addr_list().version() < b.addr_list().version(); } +class RateLimiter { +public: + explicit RateLimiter(td::uint32 capacity, double period) : capacity_(capacity), period_(period), remaining_(capacity) { + } + + bool take() { + while (remaining_ < capacity_ && increment_at_.is_in_past()) { + ++remaining_; + increment_at_ += period_; + } + if (remaining_) { + --remaining_; + if (increment_at_.is_in_past()) { + increment_at_ = td::Timestamp::in(period_); + } + return true; + } + return false; + } + + td::Timestamp ready_at() const { + if (remaining_) { + return td::Timestamp::now(); + } + return increment_at_; + } + +private: + td::uint32 capacity_; + double period_; + td::uint32 remaining_; + td::Timestamp increment_at_ = td::Timestamp::never(); +}; + } // namespace adnl } // namespace ton diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index bf919b0fd..e15e18612 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -144,7 +144,26 @@ adnl.message.part hash:int256 total_size:int offset:int data:bytes = adnl.Messag ---types--- adnl.db.node.key local_id:int256 peer_id:int256 = adnl.db.Key; -adnl.db.node.value date:int id:PublicKey addr_list:adnl.addressList priority_addr_list:adnl.addressList = adnl.db.node.Value; +adnl.db.node.value date:int id:PublicKey addr_list:adnl.addressList priority_addr_list:adnl.addressList = adnl.db.node.Value; + +adnl.stats.packets ts_start:double ts_end:double + in_packets:long in_bytes:long in_packets_channel:long in_bytes_channel:long + out_packets:long out_bytes:long out_packets_channel:long out_bytes_channel:long + out_expired_messages:long out_expired_bytes:long = adnl.stats.Packets; +adnl.stats.peerPair local_id:int256 peer_id:int256 ip_str:string + packets_recent:adnl.stats.packets packets_total:adnl.stats.packets + last_out_packet_ts:double last_in_packet_ts:double + connection_ready:Bool channel_status:int try_reinit_at:double + out_queue_messages:long out_queue_bytes:long + = adnl.stats.PeerPair; +adnl.stats.ipPackets ip_str:string packets:long = adnl.stats.IpPackets; +adnl.stats.localIdPackets ts_start:double ts_end:double + decrypted_packets:(vector adnl.stats.ipPackets) dropped_packets:(vector adnl.stats.ipPackets) = adnl.stats.LocalIdPackets; +adnl.stats.localId short_id:int256 + current_decrypt:(vector adnl.stats.ipPackets) + packets_recent:adnl.stats.localIdPackets packets_total:adnl.stats.localIdPackets + peers:(vector adnl.stats.peerPair) = adnl.stats.LocalId; +adnl.stats timestamp:double local_ids:(vector adnl.stats.localId) = adnl.Stats; ---functions--- @@ -723,6 +742,8 @@ engine.validator.setStateSerializerEnabled enabled:Bool = engine.validator.Succe engine.validator.setCollatorOptionsJson json:string = engine.validator.Success; engine.validator.getCollatorOptionsJson = engine.validator.JsonConfig; +engine.validator.getAdnlStats = adnl.Stats; + ---types--- storage.pong = storage.Pong; diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 337dd071e3f746cd822236e2dcb63c194b379de4..3d39734ef92df8a772a892713ffbe0493d8a4015 100644 GIT binary patch delta 1339 zcmZ`(O=uHA7~R#RjcJ>vNlLT%Ypa$b)|R3m_)`!IQd%XVQYwXIyF1umy1VYqiUBWz zcj@riyDxU57KO)n3Uk#vDUWMTD%P zQnD5vr)w|69|uKJTRE__Gu7bhk`rKE9mtErq*v`v138iDvyL|nM2_BA2Sjaj^7{6^ zps*4bMkk>6=2>E58`-hZA89t3;<%F=ffhIIRUtYZbfD}xAV3|q2vczO^M^eVd)BZp zIWd7f9EtMn=rUnJ%Qs9y!uG z)7=vk*sfWIY1T0|xNz1F;ymG8KhI!pm;nORfnV!C_(d2$;pS4hC0f!MF`Eqi%tYvN zrh*Yo^owj~qS2JqSw+_oHIKr}=8{5izyv7=823!22e+ZwNR9Zt@7itFe0Zu(-DGH|DI$D*ylh delta 64 zcmeBq!n)%UEAOM(`c@23z_pS0I?H4p);F8KuszUY?A?6TbiUGNmdFa`%^}G@PBAuY Q_tj%OF1kI&g>i!}0NGC$ZU6uP diff --git a/validator-engine-console/validator-engine-console-query.cpp b/validator-engine-console/validator-engine-console-query.cpp index 41721ab96..372fa8121 100644 --- a/validator-engine-console/validator-engine-console-query.cpp +++ b/validator-engine-console/validator-engine-console-query.cpp @@ -1263,3 +1263,166 @@ td::Status GetCollatorOptionsJsonQuery::receive(td::BufferSlice data) { td::TerminalIO::out() << "saved config to " << file_name_ << "\n"; return td::Status::OK(); } + +td::Status GetAdnlStatsJsonQuery::run() { + TRY_RESULT_ASSIGN(file_name_, tokenizer_.get_token()); + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status GetAdnlStatsJsonQuery::send() { + auto b = + ton::create_serialize_tl_object(); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status GetAdnlStatsJsonQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(f, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + auto s = td::json_encode(td::ToJson(*f), true); + TRY_STATUS(td::write_file(file_name_, s)); + td::TerminalIO::out() << "saved adnl stats to " << file_name_ << "\n"; + return td::Status::OK(); +} + +td::Status GetAdnlStatsQuery::run() { + TRY_STATUS(tokenizer_.check_endl()); + return td::Status::OK(); +} + +td::Status GetAdnlStatsQuery::send() { + auto b = + ton::create_serialize_tl_object(); + td::actor::send_closure(console_, &ValidatorEngineConsole::envelope_send_query, std::move(b), create_promise()); + return td::Status::OK(); +} + +td::Status GetAdnlStatsQuery::receive(td::BufferSlice data) { + TRY_RESULT_PREFIX(stats, ton::fetch_tl_object(data.as_slice(), true), + "received incorrect answer: "); + td::StringBuilder sb; + sb << "================================= ADNL STATS =================================\n"; + bool first = true; + double now = td::Clocks::system(); + for (auto &local_id : stats->local_ids_) { + if (first) { + first = false; + } else { + sb << "\n"; + } + sb << "LOCAL ID " << local_id->short_id_ << "\n"; + if (!local_id->current_decrypt_.empty()) { + std::sort(local_id->current_decrypt_.begin(), local_id->current_decrypt_.end(), + [](const ton::tl_object_ptr &a, + const ton::tl_object_ptr &b) { + return a->packets_ > b->packets_; + }); + td::uint64 total = 0; + for (auto &x : local_id->current_decrypt_) { + total += x->packets_; + } + sb << " Packets in decryptor: total=" << total; + for (auto &x : local_id->current_decrypt_) { + sb << " " << (x->ip_str_.empty() ? "unknown" : x->ip_str_) << "=" << x->packets_; + } + sb << "\n"; + } + auto print_local_id_packets = [&](const std::string &name, + std::vector> &vec) { + if (vec.empty()) { + return; + } + std::sort(vec.begin(), vec.end(), + [](const ton::tl_object_ptr &a, + const ton::tl_object_ptr &b) { + return a->packets_ > b->packets_; + }); + td::uint64 total = 0; + for (auto &x : vec) { + total += x->packets_; + } + sb << " " << name << ": total=" << total; + int cnt = 0; + for (auto &x : vec) { + ++cnt; + if (cnt >= 8) { + sb << " ..."; + break; + } + sb << " " << (x->ip_str_.empty() ? "unknown" : x->ip_str_) << "=" << x->packets_; + } + sb << "\n"; + }; + print_local_id_packets("Decrypted packets (recent)", local_id->packets_recent_->decrypted_packets_); + print_local_id_packets("Dropped packets (recent)", local_id->packets_recent_->dropped_packets_); + print_local_id_packets("Decrypted packets (total)", local_id->packets_total_->decrypted_packets_); + print_local_id_packets("Dropped packets (total)", local_id->packets_total_->dropped_packets_); + sb << " PEERS (" << local_id->peers_.size() << "):\n"; + std::sort(local_id->peers_.begin(), local_id->peers_.end(), + [](const ton::tl_object_ptr &a, + const ton::tl_object_ptr &b) { + return a->packets_recent_->in_bytes_ + a->packets_recent_->out_bytes_ > + b->packets_recent_->in_bytes_ + b->packets_recent_->out_bytes_; + }); + for (auto &peer : local_id->peers_) { + sb << " PEER " << peer->peer_id_ << "\n"; + sb << " Address: " << (peer->ip_str_.empty() ? "unknown" : peer->ip_str_) << "\n"; + sb << " Connection " << (peer->connection_ready_ ? "ready" : "not ready") << ", "; + switch (peer->channel_status_) { + case 0: + sb << "channel: none\n"; + break; + case 1: + sb << "channel: inited\n"; + break; + case 2: + sb << "channel: ready\n"; + break; + default: + sb << "\n"; + } + + auto print_packets = [&](const std::string &name, + const ton::tl_object_ptr &obj) { + if (obj->in_packets_) { + sb << " In (" << name << "): " << obj->in_packets_ << " packets (" + << td::format::as_size(obj->in_bytes_) << "), channel: " << obj->in_packets_channel_ << " packets (" + << td::format::as_size(obj->in_bytes_channel_) << ")\n"; + } + if (obj->out_packets_) { + sb << " Out (" << name << "): " << obj->out_packets_ << " packets (" + << td::format::as_size(obj->out_bytes_) << "), channel: " << obj->out_packets_channel_ << " packets (" + << td::format::as_size(obj->out_bytes_channel_) << ")\n"; + } + if (obj->out_expired_messages_) { + sb << " Out expired (" << name << "): " << obj->out_expired_messages_ << " messages (" + << td::format::as_size(obj->out_expired_bytes_) << ")\n"; + } + }; + print_packets("recent", peer->packets_recent_); + print_packets("total", peer->packets_total_); + + sb << " Last in packet: "; + if (peer->last_in_packet_ts_) { + sb << now - peer->last_in_packet_ts_ << " s ago"; + } else { + sb << "never"; + } + sb << " Last out packet: "; + if (peer->last_out_packet_ts_) { + sb << now - peer->last_out_packet_ts_ << " s ago"; + } else { + sb << "never"; + } + sb << "\n"; + if (peer->out_queue_messages_) { + sb << " Out message queue: " << peer->out_queue_messages_ << " messages (" + << td::format::as_size(peer->out_queue_bytes_) << ")\n"; + } + } + } + sb << "==============================================================================\n"; + td::TerminalIO::out() << sb.as_cslice(); + return td::Status::OK(); +} diff --git a/validator-engine-console/validator-engine-console-query.h b/validator-engine-console/validator-engine-console-query.h index 08ac1572a..6314d6199 100644 --- a/validator-engine-console/validator-engine-console-query.h +++ b/validator-engine-console/validator-engine-console-query.h @@ -1292,3 +1292,47 @@ class GetCollatorOptionsJsonQuery : public Query { private: std::string file_name_; }; + +class GetAdnlStatsJsonQuery : public Query { + public: + GetAdnlStatsJsonQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "getadnlstatsjson"; + } + static std::string get_help() { + return "getadnlstatsjson \tsave adnl stats to "; + } + std::string name() const override { + return get_name(); + } + + private: + std::string file_name_; +}; + +class GetAdnlStatsQuery : public Query { + public: + GetAdnlStatsQuery(td::actor::ActorId console, Tokenizer tokenizer) + : Query(console, std::move(tokenizer)) { + } + td::Status run() override; + td::Status send() override; + td::Status receive(td::BufferSlice data) override; + static std::string get_name() { + return "getadnlstats"; + } + static std::string get_help() { + return "getadnlstats\tdisplay adnl stats"; + } + std::string name() const override { + return get_name(); + } + + private: + std::string file_name_; +}; diff --git a/validator-engine-console/validator-engine-console.cpp b/validator-engine-console/validator-engine-console.cpp index d8a230801..1ec0f3803 100644 --- a/validator-engine-console/validator-engine-console.cpp +++ b/validator-engine-console/validator-engine-console.cpp @@ -150,6 +150,8 @@ void ValidatorEngineConsole::run() { add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); + add_query_runner(std::make_unique>()); } bool ValidatorEngineConsole::envelope_send_query(td::BufferSlice query, td::Promise promise) { diff --git a/validator-engine/validator-engine.cpp b/validator-engine/validator-engine.cpp index 11f1a9a28..31f79275e 100644 --- a/validator-engine/validator-engine.cpp +++ b/validator-engine/validator-engine.cpp @@ -3866,6 +3866,28 @@ void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getCollat } } +void ValidatorEngine::run_control_query(ton::ton_api::engine_validator_getAdnlStats &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { + if (!(perm & ValidatorEnginePermissions::vep_default)) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::error, "not authorized"))); + return; + } + if (adnl_.empty()) { + promise.set_value(create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "not started"))); + return; + } + td::actor::send_closure( + adnl_, &ton::adnl::Adnl::get_stats, + [promise = std::move(promise)](td::Result> R) mutable { + if (R.is_ok()) { + promise.set_value(ton::serialize_tl_object(R.move_as_ok(), true)); + } else { + promise.set_value( + create_control_query_error(td::Status::Error(ton::ErrorCode::notready, "failed to get adnl stats"))); + } + }); +} + void ValidatorEngine::process_control_query(td::uint16 port, ton::adnl::AdnlNodeIdShort src, ton::adnl::AdnlNodeIdShort dst, td::BufferSlice data, td::Promise promise) { diff --git a/validator-engine/validator-engine.hpp b/validator-engine/validator-engine.hpp index 7212a0b50..2e94dd1ef 100644 --- a/validator-engine/validator-engine.hpp +++ b/validator-engine/validator-engine.hpp @@ -492,6 +492,8 @@ class ValidatorEngine : public td::actor::Actor { ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); void run_control_query(ton::ton_api::engine_validator_getCollatorOptionsJson &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); + void run_control_query(ton::ton_api::engine_validator_getAdnlStats &query, td::BufferSlice data, + ton::PublicKeyHash src, td::uint32 perm, td::Promise promise); template void run_control_query(T &query, td::BufferSlice data, ton::PublicKeyHash src, td::uint32 perm, td::Promise promise) { From e35b34de22109596a54d1357dcce92d63002ba95 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Wed, 4 Sep 2024 11:38:29 +0300 Subject: [PATCH 29/37] Don't deserialize continuations in LS runSmcMethod (#1151) --- validator/impl/liteserver.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/validator/impl/liteserver.cpp b/validator/impl/liteserver.cpp index d6fad7ee2..7bedf7fe4 100644 --- a/validator/impl/liteserver.cpp +++ b/validator/impl/liteserver.cpp @@ -870,7 +870,7 @@ void LiteQuery::perform_runSmcMethod(BlockIdExt blkid, WorkchainId workchain, St vm::FakeVmStateLimits fstate(1000); // limit recursive (de)serialization calls vm::VmStateInterface::Guard guard(&fstate); auto cs = vm::load_cell_slice(res.move_as_ok()); - if (!(vm::Stack::deserialize_to(cs, stack_, 0) && cs.empty_ext())) { + if (!(vm::Stack::deserialize_to(cs, stack_, 2 /* no continuations */) && cs.empty_ext())) { fatal_error("parameter list boc cannot be deserialized as a VmStack"); return; } From cb69f307e9bd58506b048534df0593379878a949 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 5 Sep 2024 13:04:57 +0300 Subject: [PATCH 30/37] Add "lastgcdmasterchainstate" to validator stats (#1154) --- validator/db/celldb.cpp | 13 +++++++++++++ validator/db/celldb.hpp | 3 +++ validator/db/rootdb.cpp | 4 ++++ validator/db/rootdb.hpp | 1 + validator/interfaces/db.h | 1 + validator/manager.cpp | 17 +++++++++++++++++ 6 files changed, 39 insertions(+) diff --git a/validator/db/celldb.cpp b/validator/db/celldb.cpp index 463e6e34a..1701ae588 100644 --- a/validator/db/celldb.cpp +++ b/validator/db/celldb.cpp @@ -188,12 +188,17 @@ void CellDbIn::store_cell(BlockIdExt block_id, td::Ref cell, td::Promi if (!opts_->get_disable_rocksdb_stats()) { cell_db_statistics_.store_cell_time_.insert(timer.elapsed() * 1e6); } + LOG(DEBUG) << "Stored state " << block_id.to_str(); } void CellDbIn::get_cell_db_reader(td::Promise> promise) { promise.set_result(boc_->get_cell_db_reader()); } +void CellDbIn::get_last_deleted_mc_state(td::Promise promise) { + promise.set_result(last_deleted_mc_state_); +} + void CellDbIn::flush_db_stats() { if (opts_->get_disable_rocksdb_stats()) { return; @@ -320,6 +325,10 @@ void CellDbIn::gc_cont2(BlockHandle handle) { if (!opts_->get_disable_rocksdb_stats()) { cell_db_statistics_.gc_cell_time_.insert(timer.elapsed() * 1e6); } + if (handle->id().is_masterchain()) { + last_deleted_mc_state_ = handle->id().seqno(); + } + LOG(DEBUG) << "Deleted state " << handle->id().to_str(); } void CellDbIn::skip_gc() { @@ -453,6 +462,10 @@ void CellDb::get_cell_db_reader(td::Promise> p td::actor::send_closure(cell_db_, &CellDbIn::get_cell_db_reader, std::move(promise)); } +void CellDb::get_last_deleted_mc_state(td::Promise promise) { + td::actor::send_closure(cell_db_, &CellDbIn::get_last_deleted_mc_state, std::move(promise)); +} + void CellDb::start_up() { CellDbBase::start_up(); boc_ = vm::DynamicBagOfCellsDb::create(); diff --git a/validator/db/celldb.hpp b/validator/db/celldb.hpp index b3857971c..335d8a08e 100644 --- a/validator/db/celldb.hpp +++ b/validator/db/celldb.hpp @@ -61,6 +61,7 @@ class CellDbIn : public CellDbBase { void load_cell(RootHash hash, td::Promise> promise); void store_cell(BlockIdExt block_id, td::Ref cell, td::Promise> promise); void get_cell_db_reader(td::Promise> promise); + void get_last_deleted_mc_state(td::Promise promise); void migrate_cell(td::Bits256 hash); @@ -143,6 +144,7 @@ class CellDbIn : public CellDbBase { std::shared_ptr snapshot_statistics_; CellDbStatistics cell_db_statistics_; td::Timestamp statistics_flush_at_ = td::Timestamp::never(); + BlockSeqno last_deleted_mc_state_ = 0; public: class MigrationProxy : public td::actor::Actor { @@ -167,6 +169,7 @@ class CellDb : public CellDbBase { boc_->set_loader(std::make_unique(std::move(snapshot), on_load_callback_)).ensure(); } void get_cell_db_reader(td::Promise> promise); + void get_last_deleted_mc_state(td::Promise promise); CellDb(td::actor::ActorId root_db, std::string path, td::Ref opts) : root_db_(root_db), path_(path), opts_(opts) { diff --git a/validator/db/rootdb.cpp b/validator/db/rootdb.cpp index 93dcfc91f..3071f565d 100644 --- a/validator/db/rootdb.cpp +++ b/validator/db/rootdb.cpp @@ -274,6 +274,10 @@ void RootDb::get_cell_db_reader(td::Promise> p td::actor::send_closure(cell_db_, &CellDb::get_cell_db_reader, std::move(promise)); } +void RootDb::get_last_deleted_mc_state(td::Promise promise) { + td::actor::send_closure(cell_db_, &CellDb::get_last_deleted_mc_state, std::move(promise)); +} + void RootDb::store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, td::Promise promise) { td::actor::send_closure(archive_db_, &ArchiveManager::add_persistent_state, block_id, masterchain_block_id, diff --git a/validator/db/rootdb.hpp b/validator/db/rootdb.hpp index 45044e4f8..061e9add8 100644 --- a/validator/db/rootdb.hpp +++ b/validator/db/rootdb.hpp @@ -63,6 +63,7 @@ class RootDb : public Db { td::Promise> promise) override; void get_block_state(ConstBlockHandle handle, td::Promise> promise) override; void get_cell_db_reader(td::Promise> promise) override; + void get_last_deleted_mc_state(td::Promise promise) override; void store_block_handle(BlockHandle handle, td::Promise promise) override; void get_block_handle(BlockIdExt id, td::Promise promise) override; diff --git a/validator/interfaces/db.h b/validator/interfaces/db.h index 8bbf7f31f..e0d88e4e7 100644 --- a/validator/interfaces/db.h +++ b/validator/interfaces/db.h @@ -51,6 +51,7 @@ class Db : public td::actor::Actor { td::Promise> promise) = 0; virtual void get_block_state(ConstBlockHandle handle, td::Promise> promise) = 0; virtual void get_cell_db_reader(td::Promise> promise) = 0; + virtual void get_last_deleted_mc_state(td::Promise promise) = 0; virtual void store_persistent_state_file(BlockIdExt block_id, BlockIdExt masterchain_block_id, td::BufferSlice state, td::Promise promise) = 0; diff --git a/validator/manager.cpp b/validator/manager.cpp index 79927d0ef..9058be76a 100644 --- a/validator/manager.cpp +++ b/validator/manager.cpp @@ -2767,6 +2767,23 @@ void ValidatorManagerImpl::prepare_stats(td::Promiseid().seqno()](td::Result R) mutable { + TRY_RESULT_PROMISE(promise, seqno, std::move(R)); + std::string s; + if (seqno == 0) { + s = "none"; + } else if (seqno <= gc_seqno) { + s = PSTRING() << seqno << " (gc_seqno-" << (gc_seqno - seqno) << ")"; + } else { + s = PSTRING() << seqno << " (gc_seqno+" << (seqno - gc_seqno) << ")"; + } + std::vector> vec; + vec.emplace_back("lastgcdmasterchainstate", std::move(s)); + promise.set_value(std::move(vec)); + }); } if (!shard_client_.empty()) { From e32a74e9c5bfb084fa460a78c2e12718e1361637 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 5 Sep 2024 15:56:07 +0300 Subject: [PATCH 31/37] Collator: change deferring behavior when out queue size is big (#1135) --- tl/generate/scheme/ton_api.tl | 3 ++- tl/generate/scheme/ton_api.tlo | Bin 94732 -> 94848 bytes validator-engine/validator-engine.cpp | 8 +++++++ validator/impl/collator.cpp | 32 ++++++++++++++++++++++---- validator/validator.h | 5 ++++ 5 files changed, 43 insertions(+), 5 deletions(-) diff --git a/tl/generate/scheme/ton_api.tl b/tl/generate/scheme/ton_api.tl index e15e18612..10b1a9762 100644 --- a/tl/generate/scheme/ton_api.tl +++ b/tl/generate/scheme/ton_api.tl @@ -628,7 +628,8 @@ engine.validator.customOverlaysConfig overlays:(vector engine.validator.customOv engine.validator.collatorOptions deferring_enabled:Bool defer_messages_after:int defer_out_queue_size_limit:long dispatch_phase_2_max_total:int dispatch_phase_3_max_total:int - dispatch_phase_2_max_per_initiator:int dispatch_phase_3_max_per_initiator:int = engine.validator.CollatorOptions; + dispatch_phase_2_max_per_initiator:int dispatch_phase_3_max_per_initiator:int + whitelist:(vector string) prioritylist:(vector string) = engine.validator.CollatorOptions; ---functions--- ---types--- diff --git a/tl/generate/scheme/ton_api.tlo b/tl/generate/scheme/ton_api.tlo index 3d39734ef92df8a772a892713ffbe0493d8a4015..7ee378c1a1e3b7d56de9074fdc07a848e7c50ca1 100644 GIT binary patch delta 105 zcmeBq!rJhZbwi5?%Zn+mCv5HzF=%52u@t6Xv}P2Zd|{RVCy0CVn2-> parse_collator_optio } else { opts.dispatch_phase_3_max_per_initiator = {}; } + for (const std::string& s : f.whitelist_) { + TRY_RESULT(addr, block::StdAddress::parse(s)); + opts.whitelist.emplace(addr.workchain, addr.addr); + } + for (const std::string& s : f.prioritylist_) { + TRY_RESULT(addr, block::StdAddress::parse(s)); + opts.prioritylist.emplace(addr.workchain, addr.addr); + } return ref; } diff --git a/validator/impl/collator.cpp b/validator/impl/collator.cpp index f465c0f55..2cf77bfa3 100644 --- a/validator/impl/collator.cpp +++ b/validator/impl/collator.cpp @@ -3066,7 +3066,7 @@ int Collator::process_one_new_message(block::NewOutMsg msg, bool enqueue_only, R bool defer = false; if (!from_dispatch_queue) { if (deferring_messages_enabled_ && collator_opts_->deferring_enabled && !is_special && !is_special_account && - msg.msg_idx != 0) { + !collator_opts_->whitelist.count({src_wc, src_addr}) && msg.msg_idx != 0) { if (++sender_generated_messages_count_[src_addr] >= collator_opts_->defer_messages_after || out_msg_queue_size_ > defer_out_queue_size_limit_) { defer = true; @@ -3697,6 +3697,8 @@ bool Collator::process_dispatch_queue() { vm::AugmentedDictionary cur_dispatch_queue{dispatch_queue_->get_root(), 256, block::tlb::aug_DispatchQueue}; std::map, size_t> count_per_initiator; size_t total_count = 0; + auto prioritylist = collator_opts_->prioritylist; + auto prioritylist_iter = prioritylist.begin(); while (!cur_dispatch_queue.is_empty()) { block_full_ = !block_limit_status_->fits(block::ParamLimits::cl_normal); if (block_full_) { @@ -3713,9 +3715,30 @@ bool Collator::process_dispatch_queue() { return true; } StdSmcAddress src_addr; - auto account_dispatch_queue = block::get_dispatch_queue_min_lt_account(cur_dispatch_queue, src_addr); + td::Ref account_dispatch_queue; + while (!prioritylist.empty()) { + if (prioritylist_iter == prioritylist.end()) { + prioritylist_iter = prioritylist.begin(); + } + auto priority_addr = *prioritylist_iter; + if (priority_addr.first != workchain() || !is_our_address(priority_addr.second)) { + prioritylist_iter = prioritylist.erase(prioritylist_iter); + continue; + } + src_addr = priority_addr.second; + account_dispatch_queue = cur_dispatch_queue.lookup(src_addr); + if (account_dispatch_queue.is_null()) { + prioritylist_iter = prioritylist.erase(prioritylist_iter); + } else { + ++prioritylist_iter; + break; + } + } if (account_dispatch_queue.is_null()) { - return fatal_error("invalid dispatch queue in shard state"); + account_dispatch_queue = block::get_dispatch_queue_min_lt_account(cur_dispatch_queue, src_addr); + if (account_dispatch_queue.is_null()) { + return fatal_error("invalid dispatch queue in shard state"); + } } vm::Dictionary dict{64}; td::uint64 dict_size; @@ -3735,7 +3758,8 @@ bool Collator::process_dispatch_queue() { // Remove message from DispatchQueue bool ok; if (iter == 0 || - (iter == 1 && sender_generated_messages_count_[src_addr] >= collator_opts_->defer_messages_after)) { + (iter == 1 && sender_generated_messages_count_[src_addr] >= collator_opts_->defer_messages_after && + !collator_opts_->whitelist.count({workchain(), src_addr}))) { ok = cur_dispatch_queue.lookup_delete(src_addr).not_null(); } else { dict.lookup_delete(key); diff --git a/validator/validator.h b/validator/validator.h index 2171954a3..afc32f3b9 100644 --- a/validator/validator.h +++ b/validator/validator.h @@ -64,6 +64,11 @@ struct CollatorOptions : public td::CntObject { td::uint32 dispatch_phase_3_max_total = 150; td::uint32 dispatch_phase_2_max_per_initiator = 20; td::optional dispatch_phase_3_max_per_initiator; // Default - depends on out msg queue size + + // Don't defer messages from these accounts + std::set> whitelist; + // Prioritize these accounts on each phase of process_dispatch_queue + std::set> prioritylist; }; struct ValidatorManagerOptions : public td::CntObject { From 89e1cd9738fe04aba95b992490a7b44cc12e68d4 Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 5 Sep 2024 16:08:52 +0300 Subject: [PATCH 32/37] Adapt test-adnl to rate limits --- test/test-adnl.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/test/test-adnl.cpp b/test/test-adnl.cpp index 85e965a4e..45011fbc0 100644 --- a/test/test-adnl.cpp +++ b/test/test-adnl.cpp @@ -225,13 +225,19 @@ int main() { auto f = td::Clocks::system(); scheduler.run_in_context([&] { - for (td::uint32 i = 1; i <= ton::adnl::Adnl::huge_packet_max_size(); i++) { + // Don't send too many packets + // Channels are disabled, so packet rate is limited + for (td::uint32 i : {1, 2, 3, 4, 100, 500, 900}) { + remaining++; + td::actor::send_closure(adnl, &ton::adnl::Adnl::send_message, src, dst, send_packet(i)); + } + for (td::uint32 i = 1024; i <= ton::adnl::Adnl::huge_packet_max_size() /* 1024 * 8 */; i += 1024) { remaining++; td::actor::send_closure(adnl, &ton::adnl::Adnl::send_message, src, dst, send_packet(i)); } }); - auto t = td::Timestamp::in(320.0); + auto t = td::Timestamp::in(60.0); while (scheduler.run(1)) { if (!remaining) { break; @@ -241,7 +247,7 @@ int main() { } } - LOG(ERROR) << "successfully tested delivering of packets of all sizes. Time=" << (td::Clocks::system() - f); + LOG(ERROR) << "successfully tested delivering of packets of various sizes. Time=" << (td::Clocks::system() - f); scheduler.run_in_context([&] { td::actor::send_closure(network_manager, &ton::adnl::TestLoopbackNetworkManager::add_node_id, src, true, true); From 1e5d84a9d849f7897f902ac02beafb43d98f45ad Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Thu, 12 Sep 2024 09:46:38 +0300 Subject: [PATCH 33/37] mintless-proof-generator --- crypto/CMakeLists.txt | 8 + crypto/util/mintless-proof-generator.cpp | 258 +++++++++++++++++++++++ crypto/vm/db/StaticBagOfCellsDb.cpp | 10 +- tdutils/td/utils/Status.h | 2 +- 4 files changed, 272 insertions(+), 6 deletions(-) create mode 100644 crypto/util/mintless-proof-generator.cpp diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index e21f18cbe..e525f639d 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -433,6 +433,14 @@ if (WINGETOPT_FOUND) target_link_libraries_system(pow-miner wingetopt) endif() +add_executable(mintless-proof-generator util/mintless-proof-generator.cpp) +target_link_libraries(mintless-proof-generator PRIVATE ton_crypto ton_block git ${JEMALLOC_LIBRARIES}) + +if (JEMALLOC_FOUND) + target_include_directories(mintless-proof-generator PRIVATE ${JEMALLOC_INCLUDE_DIR}) + target_compile_definitions(mintless-proof-generator PRIVATE -DTON_USE_JEMALLOC=1) +endif() + set(TURN_OFF_LSAN cd .) if (TON_USE_ASAN AND NOT WIN32) set(TURN_OFF_LSAN export LSAN_OPTIONS=detect_leaks=0) diff --git a/crypto/util/mintless-proof-generator.cpp b/crypto/util/mintless-proof-generator.cpp new file mode 100644 index 000000000..49b83d931 --- /dev/null +++ b/crypto/util/mintless-proof-generator.cpp @@ -0,0 +1,258 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ + +#include "block-parse.h" +#include "block.h" +#include "td/db/utils/BlobView.h" + +#include +#include "td/utils/OptionParser.h" +#include "td/utils/Time.h" +#include "td/utils/filesystem.h" +#include "td/utils/logging.h" +#include "vm/cells/MerkleProof.h" +#include "vm/db/StaticBagOfCellsDb.h" + +#include + +void print_help() { + std::cerr << "mintless-proof-generator - generates proofs for mintless jettons\n"; + std::cerr << "Usage:\n"; + std::cerr << " mintless-proof-generator generate \tGenerate a full tree for " + ", save boc to \n"; + std::cerr << " mintless-proof-generator make_proof
\tGenerate a proof for " + "address
from tree , save boc to file \n"; + std::cerr << " mintless-proof-generator parse \tRead a tree from and output it " + "as text to \n"; + exit(2); +} + +void log_mem_stat() { + auto r_stat = td::mem_stat(); + if (r_stat.is_error()) { + LOG(WARNING) << "Memory: " << r_stat.move_as_error(); + return; + } + auto stat = r_stat.move_as_ok(); + LOG(WARNING) << "Memory: " + << "res=" << stat.resident_size_ << " (peak=" << stat.resident_size_peak_ + << ") virt=" << stat.virtual_size_ << " (peak=" << stat.virtual_size_peak_ << ")"; +} + +td::BitArray<3 + 8 + 256> address_to_key(const block::StdAddress &address) { + // addr_std$10 anycast:(Maybe Anycast) workchain_id:int8 address:bits256 = MsgAddressInt; + vm::CellBuilder cb; + cb.store_long(0b100, 3); + cb.store_long(address.workchain, 8); + cb.store_bits(address.addr.as_bitslice()); + return cb.data_bits(); +} + +struct Entry { + block::StdAddress address; + td::RefInt256 amount; + td::uint64 start_from = 0, expired_at = 0; + + td::BitArray<3 + 8 + 256> get_key() const { + return address_to_key(address); + } + + td::Ref get_value() const { + // _ amount:Coins start_from:uint48 expired_at:uint48 = AirdropItem; + vm::CellBuilder cb; + bool ok = block::tlb::t_Grams.store_integer_value(cb, *amount) && cb.store_ulong_rchk_bool(start_from, 48) && + cb.store_ulong_rchk_bool(expired_at, 48); + LOG_CHECK(ok) << "Failed to serialize AirdropItem"; + return cb.as_cellslice_ref(); + } + + static Entry parse(td::BitArray<3 + 8 + 256> key, vm::CellSlice value) { + Entry e; + td::ConstBitPtr ptr = key.bits(); + LOG_CHECK(ptr.get_uint(3) == 0b100) << "Invalid address"; + ptr.advance(3); + e.address.workchain = (ton::WorkchainId)ptr.get_int(8); + ptr.advance(8); + e.address.addr = ptr; + bool ok = block::tlb::t_Grams.as_integer_skip_to(value, e.amount) && value.fetch_uint_to(48, e.start_from) && + value.fetch_uint_to(48, e.expired_at) && value.empty_ext(); + LOG_CHECK(ok) << "Failed to parse AirdropItem"; + return e; + } +}; + +bool read_entry(std::istream &f, Entry &entry) { + std::string line; + while (std::getline(f, line)) { + std::vector v = td::full_split(line, ' '); + if (v.empty()) { + continue; + } + auto S = [&]() -> td::Status { + if (v.size() != 4) { + return td::Status::Error("Invalid line in input"); + } + TRY_RESULT_PREFIX_ASSIGN(entry.address, block::StdAddress::parse(v[0]), "Invalid address in input: "); + entry.amount = td::string_to_int256(v[1]); + if (entry.amount.is_null() || !entry.amount->is_valid() || entry.amount->sgn() < 0) { + return td::Status::Error(PSTRING() << "Invalid amount in input: " << v[1]); + } + TRY_RESULT_PREFIX_ASSIGN(entry.start_from, td::to_integer_safe(v[2]), + "Invalid start_from in input: "); + TRY_RESULT_PREFIX_ASSIGN(entry.expired_at, td::to_integer_safe(v[3]), + "Invalid expired_at in input: "); + return td::Status::OK(); + }(); + S.ensure(); + return true; + } + return false; +} + +td::Status run_generate(std::string in_filename, std::string out_filename) { + LOG(INFO) << "Generating tree from " << in_filename; + std::ifstream in_file{in_filename}; + LOG_CHECK(in_file.is_open()) << "Cannot open file " << in_filename; + + Entry entry; + vm::Dictionary dict{3 + 8 + 256}; + td::uint64 count = 0; + td::Timestamp log_at = td::Timestamp::in(5.0); + while (read_entry(in_file, entry)) { + ++count; + bool ok = dict.set(entry.get_key(), entry.get_value(), vm::DictionaryBase::SetMode::Add); + LOG_CHECK(ok) << "Failed to add entry " << entry.address.rserialize() << " (line #" << count << ")"; + if (log_at.is_in_past()) { + LOG(INFO) << "Added " << count << " entries"; + log_at = td::Timestamp::in(5.0); + } + } + LOG_CHECK(in_file.eof()) << "Failed to read file " << in_filename; + in_file.close(); + + LOG_CHECK(count != 0) << "Input is empty"; + td::Ref root = dict.get_root_cell(); + LOG(INFO) << "Total: " << count << " entries, root hash: " << root->get_hash().to_hex(); + vm::BagOfCells boc; + boc.add_root(root); + TRY_STATUS(boc.import_cells()); + LOG(INFO) << "Writing to " << out_filename; + TRY_RESULT(fd, td::FileFd::open(out_filename, td::FileFd::Write | td::FileFd::Truncate | td::FileFd::Create)); + TRY_STATUS(boc.serialize_to_file(fd, 31)); + TRY_STATUS(fd.sync()); + fd.close(); + log_mem_stat(); + return td::Status::OK(); +} + +td::Status run_make_proof(std::string in_filename, std::string s_address, std::string out_filename) { + LOG(INFO) << "Generating proof for " << s_address << ", input file is " << in_filename; + TRY_RESULT(address, block::StdAddress::parse(s_address)); + + TRY_RESULT(blob_view, td::FileBlobView::create(in_filename)); + TRY_RESULT(boc, vm::StaticBagOfCellsDbLazy::create(std::move(blob_view))); + TRY_RESULT(root, boc->get_root_cell(0)); + + vm::MerkleProofBuilder mpb{root}; + vm::Dictionary dict{mpb.root(), 3 + 8 + 256}; + auto key = address_to_key(address); + td::Ref value = dict.lookup(key); + LOG_CHECK(value.not_null()) << "No entry for address " << s_address; + Entry e = Entry::parse(key, *value); + LOG(INFO) << "Entry: address=" << e.address.workchain << ":" << e.address.addr.to_hex() + << " amount=" << e.amount->to_dec_string() << " start_from=" << e.start_from + << " expire_at=" << e.expired_at; + + TRY_RESULT(proof, mpb.extract_proof_boc()); + LOG(INFO) << "Writing proof to " << out_filename << " (" << td::format::as_size(proof.size()) << ")"; + TRY_STATUS(td::write_file(out_filename, proof)); + log_mem_stat(); + return td::Status::OK(); +} + +td::Status run_parse(std::string in_filename, std::string out_filename) { + LOG(INFO) << "Parsing " << in_filename; + std::ofstream out_file{out_filename}; + LOG_CHECK(out_file.is_open()) << "Cannot open file " << out_filename; + + TRY_RESULT(blob_view, td::FileBlobView::create(in_filename)); + TRY_RESULT(boc, vm::StaticBagOfCellsDbLazy::create(std::move(blob_view))); + TRY_RESULT(root, boc->get_root_cell(0)); + LOG(INFO) << "Root hash = " << root->get_hash().to_hex(); + vm::Dictionary dict{root, 3 + 8 + 256}; + td::Timestamp log_at = td::Timestamp::in(5.0); + td::uint64 count = 0; + bool ok = dict.check_for_each([&](td::Ref value, td::ConstBitPtr key, int key_len) { + CHECK(key_len == 3 + 8 + 256); + Entry e = Entry::parse(key, *value); + out_file << e.address.workchain << ":" << e.address.addr.to_hex() << " " << e.amount->to_dec_string() << " " + << e.start_from << " " << e.expired_at << "\n"; + LOG_CHECK(!out_file.fail()) << "Failed to write to " << out_filename; + ++count; + if (log_at.is_in_past()) { + LOG(INFO) << "Parsed " << count << " entries"; + log_at = td::Timestamp::in(5.0); + } + return true; + }); + LOG_CHECK(ok) << "Failed to parse dictionary"; + out_file.close(); + LOG_CHECK(!out_file.fail()) << "Failed to write to " << out_filename; + LOG(INFO) << "Done: " << count << " entries"; + log_mem_stat(); + return td::Status::OK(); +} + +int main(int argc, char *argv[]) { + SET_VERBOSITY_LEVEL(verbosity_INFO); + td::set_log_fatal_error_callback([](td::CSlice) { exit(2); }); + if (argc <= 1) { + print_help(); + return 2; + } + + std::string command = argv[1]; + try { + if (command == "generate") { + if (argc != 4) { + print_help(); + } + run_generate(argv[2], argv[3]).ensure(); + return 0; + } + if (command == "make_proof") { + if (argc != 5) { + print_help(); + } + run_make_proof(argv[2], argv[3], argv[4]).ensure(); + return 0; + } + if (command == "parse") { + if (argc != 4) { + print_help(); + } + run_parse(argv[2], argv[3]).ensure(); + return 0; + } + } catch (vm::VmError &e) { + LOG(FATAL) << "VM error: " << e.get_msg(); + } catch (vm::VmVirtError &e) { + LOG(FATAL) << "VM error: " << e.get_msg(); + } + + LOG(FATAL) << "Unknown command '" << command << "'"; +} \ No newline at end of file diff --git a/crypto/vm/db/StaticBagOfCellsDb.cpp b/crypto/vm/db/StaticBagOfCellsDb.cpp index 9c00a98ce..c667f334b 100644 --- a/crypto/vm/db/StaticBagOfCellsDb.cpp +++ b/crypto/vm/db/StaticBagOfCellsDb.cpp @@ -167,7 +167,7 @@ td::Result> StaticBagOfCellsDb::create_ext_cell(Cell::LevelMask level_ // class StaticBagOfCellsDbBaselineImpl : public StaticBagOfCellsDb { public: - StaticBagOfCellsDbBaselineImpl(std::vector> roots) : roots_(std::move(roots)) { + explicit StaticBagOfCellsDbBaselineImpl(std::vector> roots) : roots_(std::move(roots)) { } td::Result get_root_count() override { return roots_.size(); @@ -233,7 +233,7 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { return create_root_cell(std::move(data_cell)); }; - ~StaticBagOfCellsDbLazyImpl() { + ~StaticBagOfCellsDbLazyImpl() override { //LOG(ERROR) << deserialize_cell_cnt_ << " " << deserialize_cell_hash_cnt_; get_thread_safe_counter().add(-1); } @@ -314,11 +314,11 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { td::RwMutex::ReadLock guard; if (info_.has_index) { TRY_RESULT(new_offset_view, data_.view(td::MutableSlice(arr, info_.offset_byte_size), - info_.index_offset + idx * info_.offset_byte_size)); + info_.index_offset + (td::int64)idx * info_.offset_byte_size)); offset_view = new_offset_view; } else { guard = index_data_rw_mutex_.lock_read().move_as_ok(); - offset_view = td::Slice(index_data_).substr(idx * info_.offset_byte_size, info_.offset_byte_size); + offset_view = td::Slice(index_data_).substr((td::int64)idx * info_.offset_byte_size, info_.offset_byte_size); } CHECK(offset_view.size() == (size_t)info_.offset_byte_size); @@ -332,7 +332,7 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { } char arr[8]; TRY_RESULT(idx_view, data_.view(td::MutableSlice(arr, info_.ref_byte_size), - info_.roots_offset + root_i * info_.ref_byte_size)); + info_.roots_offset + (td::int64)root_i * info_.ref_byte_size)); CHECK(idx_view.size() == (size_t)info_.ref_byte_size); return info_.read_ref(idx_view.ubegin()); } diff --git a/tdutils/td/utils/Status.h b/tdutils/td/utils/Status.h index 8bc210dba..cff808143 100644 --- a/tdutils/td/utils/Status.h +++ b/tdutils/td/utils/Status.h @@ -80,7 +80,7 @@ TRY_RESULT_PREFIX_IMPL(TD_CONCAT(TD_CONCAT(r_, name), __LINE__), auto name, result, prefix) #define TRY_RESULT_PREFIX_ASSIGN(name, result, prefix) \ - TRY_RESULT_PREFIX_IMPL(TD_CONCAT(TD_CONCAT(r_, name), __LINE__), name, result, prefix) + TRY_RESULT_PREFIX_IMPL(TD_CONCAT(r_response, __LINE__), name, result, prefix) #define TRY_RESULT_PROMISE_PREFIX(promise_name, name, result, prefix) \ TRY_RESULT_PROMISE_PREFIX_IMPL(promise_name, TD_CONCAT(TD_CONCAT(r_, name), __LINE__), auto name, result, prefix) From 1a5bbf30f10df126d4a5b999cd572812daeac02f Mon Sep 17 00:00:00 2001 From: EmelyanenkoK Date: Thu, 12 Sep 2024 18:02:56 +0300 Subject: [PATCH 34/37] mintless-proof-generator (#1166) Co-authored-by: SpyCheese --- crypto/CMakeLists.txt | 8 + crypto/util/mintless-proof-generator.cpp | 258 +++++++++++++++++++++++ crypto/vm/db/StaticBagOfCellsDb.cpp | 10 +- tdutils/td/utils/Status.h | 2 +- 4 files changed, 272 insertions(+), 6 deletions(-) create mode 100644 crypto/util/mintless-proof-generator.cpp diff --git a/crypto/CMakeLists.txt b/crypto/CMakeLists.txt index e21f18cbe..e525f639d 100644 --- a/crypto/CMakeLists.txt +++ b/crypto/CMakeLists.txt @@ -433,6 +433,14 @@ if (WINGETOPT_FOUND) target_link_libraries_system(pow-miner wingetopt) endif() +add_executable(mintless-proof-generator util/mintless-proof-generator.cpp) +target_link_libraries(mintless-proof-generator PRIVATE ton_crypto ton_block git ${JEMALLOC_LIBRARIES}) + +if (JEMALLOC_FOUND) + target_include_directories(mintless-proof-generator PRIVATE ${JEMALLOC_INCLUDE_DIR}) + target_compile_definitions(mintless-proof-generator PRIVATE -DTON_USE_JEMALLOC=1) +endif() + set(TURN_OFF_LSAN cd .) if (TON_USE_ASAN AND NOT WIN32) set(TURN_OFF_LSAN export LSAN_OPTIONS=detect_leaks=0) diff --git a/crypto/util/mintless-proof-generator.cpp b/crypto/util/mintless-proof-generator.cpp new file mode 100644 index 000000000..49b83d931 --- /dev/null +++ b/crypto/util/mintless-proof-generator.cpp @@ -0,0 +1,258 @@ +/* + This file is part of TON Blockchain Library. + + TON Blockchain Library is free software: you can redistribute it and/or modify + it under the terms of the GNU Lesser General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + TON Blockchain Library is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Lesser General Public License for more details. + + You should have received a copy of the GNU Lesser General Public License + along with TON Blockchain Library. If not, see . +*/ + +#include "block-parse.h" +#include "block.h" +#include "td/db/utils/BlobView.h" + +#include +#include "td/utils/OptionParser.h" +#include "td/utils/Time.h" +#include "td/utils/filesystem.h" +#include "td/utils/logging.h" +#include "vm/cells/MerkleProof.h" +#include "vm/db/StaticBagOfCellsDb.h" + +#include + +void print_help() { + std::cerr << "mintless-proof-generator - generates proofs for mintless jettons\n"; + std::cerr << "Usage:\n"; + std::cerr << " mintless-proof-generator generate \tGenerate a full tree for " + ", save boc to \n"; + std::cerr << " mintless-proof-generator make_proof
\tGenerate a proof for " + "address
from tree , save boc to file \n"; + std::cerr << " mintless-proof-generator parse \tRead a tree from and output it " + "as text to \n"; + exit(2); +} + +void log_mem_stat() { + auto r_stat = td::mem_stat(); + if (r_stat.is_error()) { + LOG(WARNING) << "Memory: " << r_stat.move_as_error(); + return; + } + auto stat = r_stat.move_as_ok(); + LOG(WARNING) << "Memory: " + << "res=" << stat.resident_size_ << " (peak=" << stat.resident_size_peak_ + << ") virt=" << stat.virtual_size_ << " (peak=" << stat.virtual_size_peak_ << ")"; +} + +td::BitArray<3 + 8 + 256> address_to_key(const block::StdAddress &address) { + // addr_std$10 anycast:(Maybe Anycast) workchain_id:int8 address:bits256 = MsgAddressInt; + vm::CellBuilder cb; + cb.store_long(0b100, 3); + cb.store_long(address.workchain, 8); + cb.store_bits(address.addr.as_bitslice()); + return cb.data_bits(); +} + +struct Entry { + block::StdAddress address; + td::RefInt256 amount; + td::uint64 start_from = 0, expired_at = 0; + + td::BitArray<3 + 8 + 256> get_key() const { + return address_to_key(address); + } + + td::Ref get_value() const { + // _ amount:Coins start_from:uint48 expired_at:uint48 = AirdropItem; + vm::CellBuilder cb; + bool ok = block::tlb::t_Grams.store_integer_value(cb, *amount) && cb.store_ulong_rchk_bool(start_from, 48) && + cb.store_ulong_rchk_bool(expired_at, 48); + LOG_CHECK(ok) << "Failed to serialize AirdropItem"; + return cb.as_cellslice_ref(); + } + + static Entry parse(td::BitArray<3 + 8 + 256> key, vm::CellSlice value) { + Entry e; + td::ConstBitPtr ptr = key.bits(); + LOG_CHECK(ptr.get_uint(3) == 0b100) << "Invalid address"; + ptr.advance(3); + e.address.workchain = (ton::WorkchainId)ptr.get_int(8); + ptr.advance(8); + e.address.addr = ptr; + bool ok = block::tlb::t_Grams.as_integer_skip_to(value, e.amount) && value.fetch_uint_to(48, e.start_from) && + value.fetch_uint_to(48, e.expired_at) && value.empty_ext(); + LOG_CHECK(ok) << "Failed to parse AirdropItem"; + return e; + } +}; + +bool read_entry(std::istream &f, Entry &entry) { + std::string line; + while (std::getline(f, line)) { + std::vector v = td::full_split(line, ' '); + if (v.empty()) { + continue; + } + auto S = [&]() -> td::Status { + if (v.size() != 4) { + return td::Status::Error("Invalid line in input"); + } + TRY_RESULT_PREFIX_ASSIGN(entry.address, block::StdAddress::parse(v[0]), "Invalid address in input: "); + entry.amount = td::string_to_int256(v[1]); + if (entry.amount.is_null() || !entry.amount->is_valid() || entry.amount->sgn() < 0) { + return td::Status::Error(PSTRING() << "Invalid amount in input: " << v[1]); + } + TRY_RESULT_PREFIX_ASSIGN(entry.start_from, td::to_integer_safe(v[2]), + "Invalid start_from in input: "); + TRY_RESULT_PREFIX_ASSIGN(entry.expired_at, td::to_integer_safe(v[3]), + "Invalid expired_at in input: "); + return td::Status::OK(); + }(); + S.ensure(); + return true; + } + return false; +} + +td::Status run_generate(std::string in_filename, std::string out_filename) { + LOG(INFO) << "Generating tree from " << in_filename; + std::ifstream in_file{in_filename}; + LOG_CHECK(in_file.is_open()) << "Cannot open file " << in_filename; + + Entry entry; + vm::Dictionary dict{3 + 8 + 256}; + td::uint64 count = 0; + td::Timestamp log_at = td::Timestamp::in(5.0); + while (read_entry(in_file, entry)) { + ++count; + bool ok = dict.set(entry.get_key(), entry.get_value(), vm::DictionaryBase::SetMode::Add); + LOG_CHECK(ok) << "Failed to add entry " << entry.address.rserialize() << " (line #" << count << ")"; + if (log_at.is_in_past()) { + LOG(INFO) << "Added " << count << " entries"; + log_at = td::Timestamp::in(5.0); + } + } + LOG_CHECK(in_file.eof()) << "Failed to read file " << in_filename; + in_file.close(); + + LOG_CHECK(count != 0) << "Input is empty"; + td::Ref root = dict.get_root_cell(); + LOG(INFO) << "Total: " << count << " entries, root hash: " << root->get_hash().to_hex(); + vm::BagOfCells boc; + boc.add_root(root); + TRY_STATUS(boc.import_cells()); + LOG(INFO) << "Writing to " << out_filename; + TRY_RESULT(fd, td::FileFd::open(out_filename, td::FileFd::Write | td::FileFd::Truncate | td::FileFd::Create)); + TRY_STATUS(boc.serialize_to_file(fd, 31)); + TRY_STATUS(fd.sync()); + fd.close(); + log_mem_stat(); + return td::Status::OK(); +} + +td::Status run_make_proof(std::string in_filename, std::string s_address, std::string out_filename) { + LOG(INFO) << "Generating proof for " << s_address << ", input file is " << in_filename; + TRY_RESULT(address, block::StdAddress::parse(s_address)); + + TRY_RESULT(blob_view, td::FileBlobView::create(in_filename)); + TRY_RESULT(boc, vm::StaticBagOfCellsDbLazy::create(std::move(blob_view))); + TRY_RESULT(root, boc->get_root_cell(0)); + + vm::MerkleProofBuilder mpb{root}; + vm::Dictionary dict{mpb.root(), 3 + 8 + 256}; + auto key = address_to_key(address); + td::Ref value = dict.lookup(key); + LOG_CHECK(value.not_null()) << "No entry for address " << s_address; + Entry e = Entry::parse(key, *value); + LOG(INFO) << "Entry: address=" << e.address.workchain << ":" << e.address.addr.to_hex() + << " amount=" << e.amount->to_dec_string() << " start_from=" << e.start_from + << " expire_at=" << e.expired_at; + + TRY_RESULT(proof, mpb.extract_proof_boc()); + LOG(INFO) << "Writing proof to " << out_filename << " (" << td::format::as_size(proof.size()) << ")"; + TRY_STATUS(td::write_file(out_filename, proof)); + log_mem_stat(); + return td::Status::OK(); +} + +td::Status run_parse(std::string in_filename, std::string out_filename) { + LOG(INFO) << "Parsing " << in_filename; + std::ofstream out_file{out_filename}; + LOG_CHECK(out_file.is_open()) << "Cannot open file " << out_filename; + + TRY_RESULT(blob_view, td::FileBlobView::create(in_filename)); + TRY_RESULT(boc, vm::StaticBagOfCellsDbLazy::create(std::move(blob_view))); + TRY_RESULT(root, boc->get_root_cell(0)); + LOG(INFO) << "Root hash = " << root->get_hash().to_hex(); + vm::Dictionary dict{root, 3 + 8 + 256}; + td::Timestamp log_at = td::Timestamp::in(5.0); + td::uint64 count = 0; + bool ok = dict.check_for_each([&](td::Ref value, td::ConstBitPtr key, int key_len) { + CHECK(key_len == 3 + 8 + 256); + Entry e = Entry::parse(key, *value); + out_file << e.address.workchain << ":" << e.address.addr.to_hex() << " " << e.amount->to_dec_string() << " " + << e.start_from << " " << e.expired_at << "\n"; + LOG_CHECK(!out_file.fail()) << "Failed to write to " << out_filename; + ++count; + if (log_at.is_in_past()) { + LOG(INFO) << "Parsed " << count << " entries"; + log_at = td::Timestamp::in(5.0); + } + return true; + }); + LOG_CHECK(ok) << "Failed to parse dictionary"; + out_file.close(); + LOG_CHECK(!out_file.fail()) << "Failed to write to " << out_filename; + LOG(INFO) << "Done: " << count << " entries"; + log_mem_stat(); + return td::Status::OK(); +} + +int main(int argc, char *argv[]) { + SET_VERBOSITY_LEVEL(verbosity_INFO); + td::set_log_fatal_error_callback([](td::CSlice) { exit(2); }); + if (argc <= 1) { + print_help(); + return 2; + } + + std::string command = argv[1]; + try { + if (command == "generate") { + if (argc != 4) { + print_help(); + } + run_generate(argv[2], argv[3]).ensure(); + return 0; + } + if (command == "make_proof") { + if (argc != 5) { + print_help(); + } + run_make_proof(argv[2], argv[3], argv[4]).ensure(); + return 0; + } + if (command == "parse") { + if (argc != 4) { + print_help(); + } + run_parse(argv[2], argv[3]).ensure(); + return 0; + } + } catch (vm::VmError &e) { + LOG(FATAL) << "VM error: " << e.get_msg(); + } catch (vm::VmVirtError &e) { + LOG(FATAL) << "VM error: " << e.get_msg(); + } + + LOG(FATAL) << "Unknown command '" << command << "'"; +} \ No newline at end of file diff --git a/crypto/vm/db/StaticBagOfCellsDb.cpp b/crypto/vm/db/StaticBagOfCellsDb.cpp index 9c00a98ce..c667f334b 100644 --- a/crypto/vm/db/StaticBagOfCellsDb.cpp +++ b/crypto/vm/db/StaticBagOfCellsDb.cpp @@ -167,7 +167,7 @@ td::Result> StaticBagOfCellsDb::create_ext_cell(Cell::LevelMask level_ // class StaticBagOfCellsDbBaselineImpl : public StaticBagOfCellsDb { public: - StaticBagOfCellsDbBaselineImpl(std::vector> roots) : roots_(std::move(roots)) { + explicit StaticBagOfCellsDbBaselineImpl(std::vector> roots) : roots_(std::move(roots)) { } td::Result get_root_count() override { return roots_.size(); @@ -233,7 +233,7 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { return create_root_cell(std::move(data_cell)); }; - ~StaticBagOfCellsDbLazyImpl() { + ~StaticBagOfCellsDbLazyImpl() override { //LOG(ERROR) << deserialize_cell_cnt_ << " " << deserialize_cell_hash_cnt_; get_thread_safe_counter().add(-1); } @@ -314,11 +314,11 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { td::RwMutex::ReadLock guard; if (info_.has_index) { TRY_RESULT(new_offset_view, data_.view(td::MutableSlice(arr, info_.offset_byte_size), - info_.index_offset + idx * info_.offset_byte_size)); + info_.index_offset + (td::int64)idx * info_.offset_byte_size)); offset_view = new_offset_view; } else { guard = index_data_rw_mutex_.lock_read().move_as_ok(); - offset_view = td::Slice(index_data_).substr(idx * info_.offset_byte_size, info_.offset_byte_size); + offset_view = td::Slice(index_data_).substr((td::int64)idx * info_.offset_byte_size, info_.offset_byte_size); } CHECK(offset_view.size() == (size_t)info_.offset_byte_size); @@ -332,7 +332,7 @@ class StaticBagOfCellsDbLazyImpl : public StaticBagOfCellsDb { } char arr[8]; TRY_RESULT(idx_view, data_.view(td::MutableSlice(arr, info_.ref_byte_size), - info_.roots_offset + root_i * info_.ref_byte_size)); + info_.roots_offset + (td::int64)root_i * info_.ref_byte_size)); CHECK(idx_view.size() == (size_t)info_.ref_byte_size); return info_.read_ref(idx_view.ubegin()); } diff --git a/tdutils/td/utils/Status.h b/tdutils/td/utils/Status.h index 8bc210dba..cff808143 100644 --- a/tdutils/td/utils/Status.h +++ b/tdutils/td/utils/Status.h @@ -80,7 +80,7 @@ TRY_RESULT_PREFIX_IMPL(TD_CONCAT(TD_CONCAT(r_, name), __LINE__), auto name, result, prefix) #define TRY_RESULT_PREFIX_ASSIGN(name, result, prefix) \ - TRY_RESULT_PREFIX_IMPL(TD_CONCAT(TD_CONCAT(r_, name), __LINE__), name, result, prefix) + TRY_RESULT_PREFIX_IMPL(TD_CONCAT(r_response, __LINE__), name, result, prefix) #define TRY_RESULT_PROMISE_PREFIX(promise_name, name, result, prefix) \ TRY_RESULT_PROMISE_PREFIX_IMPL(promise_name, TD_CONCAT(TD_CONCAT(r_, name), __LINE__), auto name, result, prefix) From 76cda01af93248f12448fbe9e2a0508aab5b730e Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 13 Sep 2024 10:09:03 +0300 Subject: [PATCH 35/37] mintless-proof-generator make_all_proofs --- crypto/util/mintless-proof-generator.cpp | 180 ++++++++++++++++++++--- 1 file changed, 158 insertions(+), 22 deletions(-) diff --git a/crypto/util/mintless-proof-generator.cpp b/crypto/util/mintless-proof-generator.cpp index 49b83d931..47fd26469 100644 --- a/crypto/util/mintless-proof-generator.cpp +++ b/crypto/util/mintless-proof-generator.cpp @@ -17,27 +17,37 @@ #include "block-parse.h" #include "block.h" +#include "td/actor/core/Actor.h" #include "td/db/utils/BlobView.h" #include #include "td/utils/OptionParser.h" #include "td/utils/Time.h" +#include "td/utils/base64.h" #include "td/utils/filesystem.h" #include "td/utils/logging.h" #include "vm/cells/MerkleProof.h" #include "vm/db/StaticBagOfCellsDb.h" #include +#include + +const size_t KEY_LEN = 3 + 8 + 256; void print_help() { - std::cerr << "mintless-proof-generator - generates proofs for mintless jettons\n"; - std::cerr << "Usage:\n"; - std::cerr << " mintless-proof-generator generate \tGenerate a full tree for " - ", save boc to \n"; - std::cerr << " mintless-proof-generator make_proof
\tGenerate a proof for " - "address
from tree , save boc to file \n"; - std::cerr << " mintless-proof-generator parse \tRead a tree from and output it " - "as text to \n"; + std::cerr << "mintless-proof-generator - generates proofs for mintless jettons. Usage:\n\n"; + std::cerr << "mintless-proof-generator generate \n"; + std::cerr << " Generate a full tree for , save boc to .\n"; + std::cerr << " Input format: each line is
.\n\n"; + std::cerr << "mintless-proof-generator make_proof
.\n"; + std::cerr << " Generate a proof for address
from tree , save boc to file .\n\n"; + std::cerr << "mintless-proof-generator parse \n"; + std::cerr << " Read a tree from and output it as text to .\n"; + std::cerr << " Output format: same as input for 'generate'.\n\n"; + std::cerr << "mintless-proof-generator make_all_proofs [--threads ]\n"; + std::cerr << " Read a tree from and output proofs for all accounts to .\n"; + std::cerr << " Output format:
,\n"; + std::cerr << " Default : 1\n"; exit(2); } @@ -53,7 +63,7 @@ void log_mem_stat() { << ") virt=" << stat.virtual_size_ << " (peak=" << stat.virtual_size_peak_ << ")"; } -td::BitArray<3 + 8 + 256> address_to_key(const block::StdAddress &address) { +td::BitArray address_to_key(const block::StdAddress &address) { // addr_std$10 anycast:(Maybe Anycast) workchain_id:int8 address:bits256 = MsgAddressInt; vm::CellBuilder cb; cb.store_long(0b100, 3); @@ -62,12 +72,23 @@ td::BitArray<3 + 8 + 256> address_to_key(const block::StdAddress &address) { return cb.data_bits(); } +block::StdAddress key_to_address(const td::BitArray &key) { + block::StdAddress addr; + td::ConstBitPtr ptr = key.bits(); + LOG_CHECK(ptr.get_uint(3) == 0b100) << "Invalid address"; + ptr.advance(3); + addr.workchain = (ton::WorkchainId)ptr.get_int(8); + ptr.advance(8); + addr.addr = ptr; + return addr; +} + struct Entry { block::StdAddress address; td::RefInt256 amount; td::uint64 start_from = 0, expired_at = 0; - td::BitArray<3 + 8 + 256> get_key() const { + td::BitArray get_key() const { return address_to_key(address); } @@ -80,14 +101,9 @@ struct Entry { return cb.as_cellslice_ref(); } - static Entry parse(td::BitArray<3 + 8 + 256> key, vm::CellSlice value) { + static Entry parse(const td::BitArray &key, vm::CellSlice value) { Entry e; - td::ConstBitPtr ptr = key.bits(); - LOG_CHECK(ptr.get_uint(3) == 0b100) << "Invalid address"; - ptr.advance(3); - e.address.workchain = (ton::WorkchainId)ptr.get_int(8); - ptr.advance(8); - e.address.addr = ptr; + e.address = key_to_address(key); bool ok = block::tlb::t_Grams.as_integer_skip_to(value, e.amount) && value.fetch_uint_to(48, e.start_from) && value.fetch_uint_to(48, e.expired_at) && value.empty_ext(); LOG_CHECK(ok) << "Failed to parse AirdropItem"; @@ -129,7 +145,7 @@ td::Status run_generate(std::string in_filename, std::string out_filename) { LOG_CHECK(in_file.is_open()) << "Cannot open file " << in_filename; Entry entry; - vm::Dictionary dict{3 + 8 + 256}; + vm::Dictionary dict{KEY_LEN}; td::uint64 count = 0; td::Timestamp log_at = td::Timestamp::in(5.0); while (read_entry(in_file, entry)) { @@ -168,7 +184,7 @@ td::Status run_make_proof(std::string in_filename, std::string s_address, std::s TRY_RESULT(root, boc->get_root_cell(0)); vm::MerkleProofBuilder mpb{root}; - vm::Dictionary dict{mpb.root(), 3 + 8 + 256}; + vm::Dictionary dict{mpb.root(), KEY_LEN}; auto key = address_to_key(address); td::Ref value = dict.lookup(key); LOG_CHECK(value.not_null()) << "No entry for address " << s_address; @@ -193,11 +209,11 @@ td::Status run_parse(std::string in_filename, std::string out_filename) { TRY_RESULT(boc, vm::StaticBagOfCellsDbLazy::create(std::move(blob_view))); TRY_RESULT(root, boc->get_root_cell(0)); LOG(INFO) << "Root hash = " << root->get_hash().to_hex(); - vm::Dictionary dict{root, 3 + 8 + 256}; + vm::Dictionary dict{root, KEY_LEN}; td::Timestamp log_at = td::Timestamp::in(5.0); td::uint64 count = 0; bool ok = dict.check_for_each([&](td::Ref value, td::ConstBitPtr key, int key_len) { - CHECK(key_len == 3 + 8 + 256); + CHECK(key_len == KEY_LEN); Entry e = Entry::parse(key, *value); out_file << e.address.workchain << ":" << e.address.addr.to_hex() << " " << e.amount->to_dec_string() << " " << e.start_from << " " << e.expired_at << "\n"; @@ -212,7 +228,108 @@ td::Status run_parse(std::string in_filename, std::string out_filename) { LOG_CHECK(ok) << "Failed to parse dictionary"; out_file.close(); LOG_CHECK(!out_file.fail()) << "Failed to write to " << out_filename; - LOG(INFO) << "Done: " << count << " entries"; + LOG(INFO) << "Written " << count << " entries to " << out_filename; + log_mem_stat(); + return td::Status::OK(); +} + +class MakeAllProofsActor : public td::actor::core::Actor { + public: + MakeAllProofsActor(std::string in_filename, std::string out_filename, int max_workers) + : in_filename_(in_filename), out_filename_(out_filename), max_workers_(max_workers) { + } + + void start_up() override { + auto S = [&]() -> td::Status { + out_file_.open(out_filename_); + LOG_CHECK(out_file_.is_open()) << "Cannot open file " << out_filename_; + LOG(INFO) << "Reading " << in_filename_; + TRY_RESULT(blob_view, td::FileBlobView::create(in_filename_)); + TRY_RESULT(boc, vm::StaticBagOfCellsDbLazy::create(std::move(blob_view))); + TRY_RESULT(root, boc->get_root_cell(0)); + LOG(INFO) << "Root hash = " << root->get_hash().to_hex(); + dict_ = vm::Dictionary{root, KEY_LEN}; + return td::Status::OK(); + }(); + S.ensure(); + run(); + alarm_timestamp() = td::Timestamp::in(5.0); + } + + void alarm() override { + alarm_timestamp() = td::Timestamp::in(5.0); + LOG(INFO) << "Processed " << written_count_ << " entries"; + } + + void run() { + for (auto it = pending_results_.begin(); it != pending_results_.end() && !it->second.empty();) { + out_file_ << it->second << "\n"; + LOG_CHECK(!out_file_.fail()) << "Failed to write to " << out_filename_; + it = pending_results_.erase(it); + ++written_count_; + } + while (active_workers_ < max_workers_ && !eof_) { + td::Ref value = dict_.lookup_nearest_key(current_key_, true, current_idx_ == 0); + if (value.is_null()) { + eof_ = true; + break; + } + run_worker(current_key_, current_idx_); + ++current_idx_; + ++active_workers_; + } + if (eof_ && active_workers_ == 0) { + out_file_.close(); + LOG_CHECK(!out_file_.fail()) << "Failed to write to " << out_filename_; + LOG(INFO) << "Written " << written_count_ << " entries to " << out_filename_; + stop(); + td::actor::SchedulerContext::get()->stop(); + } + } + + void run_worker(td::BitArray key, td::uint64 idx) { + pending_results_[idx] = ""; + ton::delay_action( + [SelfId = actor_id(this), key, idx, root = dict_.get_root_cell()]() { + vm::MerkleProofBuilder mpb{root}; + CHECK(vm::Dictionary(mpb.root(), KEY_LEN).lookup(key).not_null()); + auto r_proof = mpb.extract_proof_boc(); + r_proof.ensure(); + block::StdAddress addr = key_to_address(key); + std::string result = PSTRING() << addr.workchain << ":" << addr.addr.to_hex() << "," + << td::base64_encode(r_proof.move_as_ok()); + td::actor::send_closure(SelfId, &MakeAllProofsActor::on_result, idx, std::move(result)); + }, + td::Timestamp::now()); + } + + void on_result(td::uint64 idx, std::string result) { + pending_results_[idx] = std::move(result); + --active_workers_; + run(); + } + + private: + std::string in_filename_, out_filename_; + int max_workers_; + + std::ofstream out_file_; + vm::Dictionary dict_{KEY_LEN}; + td::BitArray current_key_ = td::BitArray::zero(); + td::uint64 current_idx_ = 0; + bool eof_ = false; + int active_workers_ = 0; + + std::map pending_results_; + td::uint64 written_count_ = 0; +}; + +td::Status run_make_all_proofs(std::string in_filename, std::string out_filename, int threads) { + td::actor::Scheduler scheduler({(size_t)threads}); + scheduler.run_in_context( + [&] { td::actor::create_actor("proofs", in_filename, out_filename, threads).release(); }); + while (scheduler.run(1)) { + } log_mem_stat(); return td::Status::OK(); } @@ -248,6 +365,25 @@ int main(int argc, char *argv[]) { run_parse(argv[2], argv[3]).ensure(); return 0; } + if (command == "make_all_proofs") { + std::vector args; + int threads = 1; + for (int i = 2; i < argc; ++i) { + if (!strcmp(argv[i], "--threads")) { + ++i; + auto r = td::to_integer_safe(td::as_slice(argv[i])); + LOG_CHECK(r.is_ok() && r.ok() >= 1 && r.ok() <= 127) << " should be in [1..127]"; + threads = r.move_as_ok(); + } else { + args.push_back(argv[i]); + } + } + if (args.size() != 2) { + print_help(); + } + run_make_all_proofs(args[0], args[1], threads).ensure(); + return 0; + } } catch (vm::VmError &e) { LOG(FATAL) << "VM error: " << e.get_msg(); } catch (vm::VmVirtError &e) { From b304b1c7be896b0ae877bc835d4662985e15cf5f Mon Sep 17 00:00:00 2001 From: SpyCheese Date: Fri, 13 Sep 2024 20:47:30 +0300 Subject: [PATCH 36/37] LS getDispatchQueueInfo and getDispatchQueueMessages methods (#1161) * liteServer.getDispatchQueueInfo query * Fix getting min/max lt * LS getDispatchQueueMessages method --- lite-client/lite-client.cpp | 96 +++++++++++- lite-client/lite-client.h | 7 + tl-utils/lite-utils.cpp | 2 + tl/generate/scheme/lite_api.tl | 8 + tl/generate/scheme/lite_api.tlo | Bin 18356 -> 20256 bytes validator/impl/liteserver.cpp | 249 ++++++++++++++++++++++++++++++++ validator/impl/liteserver.hpp | 5 + 7 files changed, 364 insertions(+), 3 deletions(-) diff --git a/lite-client/lite-client.cpp b/lite-client/lite-client.cpp index da2fd6ff9..a6557b492 100644 --- a/lite-client/lite-client.cpp +++ b/lite-client/lite-client.cpp @@ -974,6 +974,11 @@ bool TestNode::show_help(std::string command) { "into files .boc\n" "complaintprice \tComputes the price (in nanograms) for creating a complaint\n" "msgqueuesizes\tShows current sizes of outbound message queues in all shards\n" + "dispatchqueueinfo \tShows list of account dispatch queue of a block\n" + "dispatchqueuemessages []\tShows deferred messages from account , lt > " + "\n" + "dispatchqueuemessagesall [ []]\tShows messages from dispatch queue of a " + "block, starting after , \n" "known\tShows the list of all known block ids\n" "knowncells\tShows the list of hashes of all known (cached) cells\n" "dumpcell \nDumps a cached cell by a prefix of its hash\n" @@ -988,9 +993,9 @@ bool TestNode::show_help(std::string command) { bool TestNode::do_parse_line() { ton::WorkchainId workchain = ton::masterchainId; // change to basechain later int addr_ext = 0; - ton::StdSmcAddress addr{}; + ton::StdSmcAddress addr = ton::StdSmcAddress::zero(); ton::BlockIdExt blkid{}; - ton::LogicalTime lt{}; + ton::LogicalTime lt = 0; ton::Bits256 hash{}; ton::ShardIdFull shard{}; ton::BlockSeqno seqno{}; @@ -1118,6 +1123,16 @@ bool TestNode::do_parse_line() { set_error(get_complaint_price(expire_in, filename)); } else if (word == "msgqueuesizes") { return get_msg_queue_sizes(); + } else if (word == "dispatchqueueinfo") { + return parse_block_id_ext(blkid) && seekeoln() && get_dispatch_queue_info(blkid); + } else if (word == "dispatchqueuemessages" || word == "dispatchqueuemessagesall") { + bool one_account = word == "dispatchqueuemessages"; + if (!parse_block_id_ext(blkid)) { + return false; + } + workchain = blkid.id.workchain; + return ((!one_account && seekeoln()) || parse_account_addr(workchain, addr)) && (seekeoln() || parse_lt(lt)) && + seekeoln() && get_dispatch_queue_messages(blkid, workchain, addr, lt, one_account); } else if (word == "known") { return eoln() && show_new_blkids(true); } else if (word == "knowncells") { @@ -1645,6 +1660,81 @@ void TestNode::got_msg_queue_sizes(ton::tl_object_ptrext_msg_queue_size_limit_ << std::endl; } +bool TestNode::get_dispatch_queue_info(ton::BlockIdExt block_id) { + td::TerminalIO::out() << "Dispatch queue in block: " << block_id.id.to_str() << std::endl; + return get_dispatch_queue_info_cont(block_id, true, td::Bits256::zero()); +} + +bool TestNode::get_dispatch_queue_info_cont(ton::BlockIdExt block_id, bool first, td::Bits256 after_addr) { + auto q = ton::create_serialize_tl_object( + first ? 0 : 2, ton::create_tl_lite_block_id(block_id), after_addr, 32, false); + return envelope_send_query(std::move(q), [=, Self = actor_id(this)](td::Result res) -> void { + if (res.is_error()) { + LOG(ERROR) << "liteServer.getDispatchQueueInfo error: " << res.move_as_error(); + return; + } + auto F = ton::fetch_tl_object(res.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.getDispatchQueueInfo"; + return; + } + td::actor::send_closure_later(Self, &TestNode::got_dispatch_queue_info, block_id, F.move_as_ok()); + }); +} + +void TestNode::got_dispatch_queue_info(ton::BlockIdExt block_id, + ton::tl_object_ptr info) { + for (auto& acc : info->account_dispatch_queues_) { + td::TerminalIO::out() << block_id.id.workchain << ":" << acc->addr_.to_hex() << " : size=" << acc->size_ + << " lt=" << acc->min_lt_ << ".." << acc->max_lt_ << std::endl; + } + if (info->complete_) { + td::TerminalIO::out() << "Done" << std::endl; + return; + } + get_dispatch_queue_info_cont(block_id, false, info->account_dispatch_queues_.back()->addr_); +} + +bool TestNode::get_dispatch_queue_messages(ton::BlockIdExt block_id, ton::WorkchainId wc, ton::StdSmcAddress addr, + ton::LogicalTime lt, bool one_account) { + if (wc != block_id.id.workchain) { + return set_error("workchain mismatch"); + } + auto q = ton::create_serialize_tl_object( + one_account ? 2 : 0, ton::create_tl_lite_block_id(block_id), addr, lt, 64, false, one_account, false); + return envelope_send_query(std::move(q), [=, Self = actor_id(this)](td::Result res) -> void { + if (res.is_error()) { + LOG(ERROR) << "liteServer.getDispatchQueueMessages error: " << res.move_as_error(); + return; + } + auto F = ton::fetch_tl_object(res.move_as_ok(), true); + if (F.is_error()) { + LOG(ERROR) << "cannot parse answer to liteServer.getDispatchQueueMessages"; + return; + } + td::actor::send_closure_later(Self, &TestNode::got_dispatch_queue_messages, F.move_as_ok()); + }); +} + +void TestNode::got_dispatch_queue_messages(ton::tl_object_ptr msgs) { + td::TerminalIO::out() << "Dispatch queue messages (" << msgs->messages_.size() << "):\n"; + int count = 0; + for (auto& m : msgs->messages_) { + auto& meta = m->metadata_; + td::TerminalIO::out() << "Msg #" << ++count << ": " << msgs->id_->workchain_ << ":" << m->addr_.to_hex() << " " + << m->lt_ << " : " + << (meta->initiator_->workchain_ == ton::workchainInvalid + ? "[ no metadata ]" + : block::MsgMetadata{(td::uint32)meta->depth_, meta->initiator_->workchain_, + meta->initiator_->id_, (ton::LogicalTime)meta->initiator_lt_} + .to_str()) + << "\n"; + } + if (!msgs->complete_) { + td::TerminalIO::out() << "(incomplete list)\n"; + } +} + bool TestNode::dns_resolve_start(ton::WorkchainId workchain, ton::StdSmcAddress addr, ton::BlockIdExt blkid, std::string domain, td::Bits256 cat, int mode) { if (domain.size() >= 2 && domain[0] == '"' && domain.back() == '"') { @@ -4322,7 +4412,7 @@ int main(int argc, char* argv[]) { }); p.add_option('V', "version", "shows lite-client build information", [&]() { std::cout << "lite-client build information: [ Commit: " << GitMetadata::CommitSHA1() << ", Date: " << GitMetadata::CommitDate() << "]\n"; - + std::exit(0); }); p.add_option('i', "idx", "set liteserver idx", [&](td::Slice arg) { diff --git a/lite-client/lite-client.h b/lite-client/lite-client.h index 17680f448..6fb7f9ab5 100644 --- a/lite-client/lite-client.h +++ b/lite-client/lite-client.h @@ -307,6 +307,13 @@ class TestNode : public td::actor::Actor { unsigned refs, td::Bits256 chash, std::string filename); bool get_msg_queue_sizes(); void got_msg_queue_sizes(ton::tl_object_ptr f); + bool get_dispatch_queue_info(ton::BlockIdExt block_id); + bool get_dispatch_queue_info_cont(ton::BlockIdExt block_id, bool first, td::Bits256 after_addr); + void got_dispatch_queue_info(ton::BlockIdExt block_id, + ton::tl_object_ptr info); + bool get_dispatch_queue_messages(ton::BlockIdExt block_id, ton::WorkchainId wc, ton::StdSmcAddress addr, + ton::LogicalTime lt, bool one_account); + void got_dispatch_queue_messages(ton::tl_object_ptr msgs); bool cache_cell(Ref cell); bool list_cached_cells() const; bool dump_cached_cell(td::Slice hash_pfx, td::Slice type_name = {}); diff --git a/tl-utils/lite-utils.cpp b/tl-utils/lite-utils.cpp index daa3dbaf0..387474cfa 100644 --- a/tl-utils/lite-utils.cpp +++ b/tl-utils/lite-utils.cpp @@ -160,6 +160,8 @@ std::string lite_query_name_by_id(int id) { {lite_api::liteServer_getShardBlockProof::ID, "getShardBlockProof"}, {lite_api::liteServer_getOutMsgQueueSizes::ID, "getOutMsgQueueSizes"}, {lite_api::liteServer_getBlockOutMsgQueueSize::ID, "getBlockOutMsgQueueSize"}, + {lite_api::liteServer_getDispatchQueueInfo::ID, "getDispatchQueueInfo"}, + {lite_api::liteServer_getDispatchQueueMessages::ID, "getDispatchQueueMessages"}, {lite_api::liteServer_nonfinal_getCandidate::ID, "nonfinal.getCandidate"}, {lite_api::liteServer_nonfinal_getValidatorGroups::ID, "nonfinal.getValidatorGroups"}}; auto it = names.find(id); diff --git a/tl/generate/scheme/lite_api.tl b/tl/generate/scheme/lite_api.tl index 879d7ff4a..b00951da4 100644 --- a/tl/generate/scheme/lite_api.tl +++ b/tl/generate/scheme/lite_api.tl @@ -61,6 +61,11 @@ liteServer.lookupBlockResult id:tonNode.blockIdExt mode:# mc_block_id:tonNode.bl liteServer.outMsgQueueSize id:tonNode.blockIdExt size:int = liteServer.OutMsgQueueSize; liteServer.outMsgQueueSizes shards:(vector liteServer.outMsgQueueSize) ext_msg_queue_size_limit:int = liteServer.OutMsgQueueSizes; liteServer.blockOutMsgQueueSize mode:# id:tonNode.blockIdExt size:long proof:mode.0?bytes = liteServer.BlockOutMsgQueueSize; +liteServer.accountDispatchQueueInfo addr:int256 size:long min_lt:long max_lt:long = liteServer.AccountDispatchQueueInfo; +liteServer.dispatchQueueInfo mode:# id:tonNode.blockIdExt account_dispatch_queues:(vector liteServer.accountDispatchQueueInfo) complete:Bool proof:mode.0?bytes = liteServer.DispatchQueueInfo; +liteServer.dispatchQueueMessage addr:int256 lt:long hash:int256 metadata:liteServer.transactionMetadata = liteServer.DispatchQueueMessage; +liteServer.dispatchQueueMessages mode:# id:tonNode.blockIdExt messages:(vector liteServer.dispatchQueueMessage) complete:Bool + proof:mode.0?bytes messages_boc:mode.2?bytes = liteServer.DispatchQueueMessages; liteServer.debug.verbosity value:int = liteServer.debug.Verbosity; @@ -100,6 +105,9 @@ liteServer.getLibrariesWithProof id:tonNode.blockIdExt mode:# library_list:(vect liteServer.getShardBlockProof id:tonNode.blockIdExt = liteServer.ShardBlockProof; liteServer.getOutMsgQueueSizes mode:# wc:mode.0?int shard:mode.0?long = liteServer.OutMsgQueueSizes; liteServer.getBlockOutMsgQueueSize mode:# id:tonNode.blockIdExt want_proof:mode.0?true = liteServer.BlockOutMsgQueueSize; +liteServer.getDispatchQueueInfo mode:# id:tonNode.blockIdExt after_addr:mode.1?int256 max_accounts:int want_proof:mode.0?true = liteServer.DispatchQueueInfo; +liteServer.getDispatchQueueMessages mode:# id:tonNode.blockIdExt addr:int256 after_lt:long max_messages:int + want_proof:mode.0?true one_account:mode.1?true messages_boc:mode.2?true = liteServer.DispatchQueueMessages; liteServer.nonfinal.getValidatorGroups mode:# wc:mode.0?int shard:mode.0?long = liteServer.nonfinal.ValidatorGroups; liteServer.nonfinal.getCandidate id:liteServer.nonfinal.candidateId = liteServer.nonfinal.Candidate; diff --git a/tl/generate/scheme/lite_api.tlo b/tl/generate/scheme/lite_api.tlo index 6ece1d20f640117cb39b71f4875f293a3f1b081b..c576a7f458bc48ea7b02c0adc0f6584cb12608b1 100644 GIT binary patch delta 929 zcmdne&$wV7Bk!Zx`c@23;INTbok?T&s-W4*IhiG?!Kp=MsYQB@$;tVpc_l8H#RZ8a z$r*vAsimo&d1?6*FGy@QVd`O0ykI04D}$r~L*-;f7HO4N%12t{ajEi6EiO(>PX!vO zVp!?mtw2ECBo>*?2UslF867wOXvwz()dD902WW)AT0wnc(R0!!Q>b*9WRhRSWH}%7%0x}Zn5tYdgBw6`jvhhjz$qWpWrBzf}K-xDKNL5HODsO&i{F_DhbM`Yv zperform_getBlockOutMsgQueueSize(q.mode_, create_block_id(q.id_)); }, + [&](lite_api::liteServer_getDispatchQueueInfo& q) { + this->perform_getDispatchQueueInfo(q.mode_, create_block_id(q.id_), q.after_addr_, q.max_accounts_); + }, + [&](lite_api::liteServer_getDispatchQueueMessages& q) { + this->perform_getDispatchQueueMessages(q.mode_, create_block_id(q.id_), q.addr_, + std::max(q.after_lt_, 0), q.max_messages_); + }, [&](auto& obj) { this->abort_query(td::Status::Error(ErrorCode::protoviolation, "unknown query")); })); } @@ -3432,6 +3439,248 @@ void LiteQuery::finish_getBlockOutMsgQueueSize() { finish_query(std::move(b)); } +void LiteQuery::perform_getDispatchQueueInfo(int mode, BlockIdExt blkid, StdSmcAddress after_addr, int max_accounts) { + LOG(INFO) << "started a getDispatchQueueInfo(" << blkid.to_str() << ", " << mode << ") liteserver query"; + mode_ = mode; + if (!blkid.is_valid_full()) { + fatal_error("invalid BlockIdExt"); + return; + } + if (max_accounts <= 0) { + fatal_error("invalid max_accounts"); + return; + } + set_continuation([=]() -> void { finish_getDispatchQueueInfo(after_addr, max_accounts); }); + request_block_data_state(blkid); +} + +void LiteQuery::finish_getDispatchQueueInfo(StdSmcAddress after_addr, int max_accounts) { + LOG(INFO) << "completing getDispatchQueueInfo() query"; + bool with_proof = mode_ & 1; + Ref state_root = state_->root_cell(); + vm::MerkleProofBuilder pb; + if (with_proof) { + pb = vm::MerkleProofBuilder{state_root}; + state_root = pb.root(); + } + + std::unique_ptr dispatch_queue; + + block::gen::ShardStateUnsplit::Record sstate; + block::gen::OutMsgQueueInfo::Record out_msg_queue_info; + block::gen::OutMsgQueueExtra::Record out_msg_queue_extra; + if (!tlb::unpack_cell(state_root, sstate) || !tlb::unpack_cell(sstate.out_msg_queue_info, out_msg_queue_info)) { + fatal_error("cannot unpack shard state"); + return; + } + vm::CellSlice& extra_slice = out_msg_queue_info.extra.write(); + if (extra_slice.fetch_long(1)) { + if (!tlb::unpack(extra_slice, out_msg_queue_extra)) { + fatal_error("cannot unpack OutMsgQueueExtra"); + return; + } + dispatch_queue = std::make_unique(out_msg_queue_extra.dispatch_queue, 256, + block::tlb::aug_DispatchQueue); + } else { + dispatch_queue = std::make_unique(256, block::tlb::aug_DispatchQueue); + } + + int remaining = std::min(max_accounts, 64); + bool complete = false; + std::vector> result; + bool allow_eq; + if (mode_ & 2) { + allow_eq = false; + } else { + allow_eq = true; + after_addr = td::Bits256::zero(); + } + while (true) { + auto value = dispatch_queue->extract_value(dispatch_queue->lookup_nearest_key(after_addr, true, allow_eq)); + allow_eq = false; + if (value.is_null()) { + complete = true; + break; + } + if (remaining == 0) { + break; + } + --remaining; + StdSmcAddress addr = after_addr; + vm::Dictionary dict{64}; + td::uint64 dict_size; + if (!block::unpack_account_dispatch_queue(value, dict, dict_size)) { + fatal_error(PSTRING() << "invalid account dispatch queue for account " << addr.to_hex()); + return; + } + CHECK(dict_size > 0); + td::BitArray<64> min_lt, max_lt; + dict.get_minmax_key(min_lt.bits(), 64, false, false); + dict.get_minmax_key(max_lt.bits(), 64, true, false); + result.push_back(create_tl_object(addr, dict_size, min_lt.to_ulong(), + max_lt.to_ulong())); + } + + td::BufferSlice proof; + if (with_proof) { + Ref proof1, proof2; + if (!make_state_root_proof(proof1)) { + return; + } + if (!pb.extract_proof_to(proof2)) { + fatal_error("unknown error creating Merkle proof"); + return; + } + auto r_proof = vm::std_boc_serialize_multi({std::move(proof1), std::move(proof2)}); + if (r_proof.is_error()) { + fatal_error(r_proof.move_as_error()); + return; + } + proof = r_proof.move_as_ok(); + } + LOG(INFO) << "getDispatchQueueInfo(" << blk_id_.to_str() << ", " << mode_ << ") query completed"; + auto b = ton::create_serialize_tl_object( + mode_, ton::create_tl_lite_block_id(blk_id_), std::move(result), complete, std::move(proof)); + finish_query(std::move(b)); +} + +void LiteQuery::perform_getDispatchQueueMessages(int mode, BlockIdExt blkid, StdSmcAddress addr, LogicalTime lt, + int max_messages) { + LOG(INFO) << "started a getDispatchQueueMessages(" << blkid.to_str() << ", " << mode << ") liteserver query"; + mode_ = mode; + if (!blkid.is_valid_full()) { + fatal_error("invalid BlockIdExt"); + return; + } + if (max_messages <= 0) { + fatal_error("invalid max_messages"); + return; + } + set_continuation([=]() -> void { finish_getDispatchQueueMessages(addr, lt, max_messages); }); + request_block_data_state(blkid); +} + +void LiteQuery::finish_getDispatchQueueMessages(StdSmcAddress addr, LogicalTime lt, int max_messages) { + LOG(INFO) << "completing getDispatchQueueMessages() query"; + bool with_proof = mode_ & lite_api::liteServer_getDispatchQueueMessages::WANT_PROOF_MASK; + bool one_account = mode_ & lite_api::liteServer_getDispatchQueueMessages::ONE_ACCOUNT_MASK; + bool with_messages_boc = mode_ & lite_api::liteServer_getDispatchQueueMessages::MESSAGES_BOC_MASK; + Ref state_root = state_->root_cell(); + vm::MerkleProofBuilder pb; + if (with_proof) { + pb = vm::MerkleProofBuilder{state_root}; + state_root = pb.root(); + } + + std::unique_ptr dispatch_queue; + + block::gen::ShardStateUnsplit::Record sstate; + block::gen::OutMsgQueueInfo::Record out_msg_queue_info; + block::gen::OutMsgQueueExtra::Record out_msg_queue_extra; + if (!tlb::unpack_cell(state_root, sstate) || !tlb::unpack_cell(sstate.out_msg_queue_info, out_msg_queue_info)) { + fatal_error("cannot unpack shard state"); + return; + } + vm::CellSlice& extra_slice = out_msg_queue_info.extra.write(); + if (extra_slice.fetch_long(1)) { + if (!tlb::unpack(extra_slice, out_msg_queue_extra)) { + fatal_error("cannot unpack OutMsgQueueExtra"); + return; + } + dispatch_queue = std::make_unique(out_msg_queue_extra.dispatch_queue, 256, + block::tlb::aug_DispatchQueue); + } else { + dispatch_queue = std::make_unique(256, block::tlb::aug_DispatchQueue); + } + + int remaining = std::min(max_messages, with_messages_boc ? 16 : 64); + bool complete = false; + std::vector> result; + std::vector> message_roots; + td::Bits256 orig_addr = addr; + bool first = true; + while (remaining > 0) { + auto value = dispatch_queue->extract_value(dispatch_queue->lookup_nearest_key(addr, true, first)); + if (value.is_null() || (one_account && addr != orig_addr)) { + complete = true; + break; + } + vm::Dictionary account_queue{64}; + td::uint64 dict_size; + if (!block::unpack_account_dispatch_queue(value, account_queue, dict_size)) { + fatal_error(PSTRING() << "invalid account dispatch queue for account " << addr.to_hex()); + return; + } + CHECK(dict_size > 0); + while (true) { + td::BitArray<64> lt_key; + lt_key.store_ulong(lt); + auto value2 = account_queue.lookup_nearest_key(lt_key, true, false); + if (value2.is_null()) { + break; + } + lt = lt_key.to_ulong(); + if (remaining == 0) { + break; + } + --remaining; + auto msg_env = value2->prefetch_ref(); + block::tlb::MsgEnvelope::Record_std env; + if (msg_env.is_null() || !tlb::unpack_cell(msg_env, env)) { + fatal_error(PSTRING() << "invalid message in dispatch queue for account " << addr.to_hex() << ", lt " << lt); + return; + } + message_roots.push_back(env.msg); + tl_object_ptr metadata_tl; + if (env.metadata) { + auto& metadata = env.metadata.value(); + metadata_tl = create_tl_object( + 0, metadata.depth, + create_tl_object(metadata.initiator_wc, metadata.initiator_addr), + metadata.initiator_lt); + } else { + metadata_tl = create_tl_object( + 0, -1, create_tl_object(workchainInvalid, td::Bits256::zero()), -1); + } + result.push_back(create_tl_object(addr, lt, env.msg->get_hash().bits(), + std::move(metadata_tl))); + } + first = false; + lt = 0; + } + + td::BufferSlice proof; + if (with_proof) { + Ref proof1, proof2; + if (!make_state_root_proof(proof1)) { + return; + } + if (!pb.extract_proof_to(proof2)) { + fatal_error("unknown error creating Merkle proof"); + return; + } + auto r_proof = vm::std_boc_serialize_multi({std::move(proof1), std::move(proof2)}); + if (r_proof.is_error()) { + fatal_error(r_proof.move_as_error()); + return; + } + proof = r_proof.move_as_ok(); + } + td::BufferSlice messages_boc; + if (with_messages_boc) { + auto r_messages_boc = vm::std_boc_serialize_multi(std::move(message_roots)); + if (r_messages_boc.is_error()) { + fatal_error(r_messages_boc.move_as_error()); + return; + } + messages_boc = std::move(messages_boc); + } + LOG(INFO) << "getDispatchQueueMessages(" << blk_id_.to_str() << ", " << mode_ << ") query completed"; + auto b = ton::create_serialize_tl_object( + mode_, ton::create_tl_lite_block_id(blk_id_), std::move(result), complete, std::move(proof), + std::move(messages_boc)); + finish_query(std::move(b)); +} void LiteQuery::perform_nonfinal_getCandidate(td::Bits256 source, BlockIdExt blkid, td::Bits256 collated_data_hash) { LOG(INFO) << "started a nonfinal.getCandidate liteserver query"; diff --git a/validator/impl/liteserver.hpp b/validator/impl/liteserver.hpp index 2d75dc61c..447e1dad4 100644 --- a/validator/impl/liteserver.hpp +++ b/validator/impl/liteserver.hpp @@ -172,6 +172,11 @@ class LiteQuery : public td::actor::Actor { void continue_getOutMsgQueueSizes(td::optional shard, Ref state); void perform_getBlockOutMsgQueueSize(int mode, BlockIdExt blkid); void finish_getBlockOutMsgQueueSize(); + void perform_getDispatchQueueInfo(int mode, BlockIdExt blkid, StdSmcAddress after_addr, int max_accounts); + void finish_getDispatchQueueInfo(StdSmcAddress after_addr, int max_accounts); + void perform_getDispatchQueueMessages(int mode, BlockIdExt blkid, StdSmcAddress addr, LogicalTime lt, + int max_messages); + void finish_getDispatchQueueMessages(StdSmcAddress addr, LogicalTime lt, int max_messages); void perform_nonfinal_getCandidate(td::Bits256 source, BlockIdExt blkid, td::Bits256 collated_data_hash); void perform_nonfinal_getValidatorGroups(int mode, ShardIdFull shard); From eea95aeebb49de6f2b204bab96b3f497ae4f2c43 Mon Sep 17 00:00:00 2001 From: neodix42 Date: Fri, 13 Sep 2024 21:48:38 +0400 Subject: [PATCH 37/37] Add tonlib libraries for Android to release (#1169) * add android tonlib artifacts to release * change final artifacts access rights --- .../workflows/build-ton-linux-android-tonlib.yml | 2 +- .github/workflows/create-release.yml | 16 ++++++++++++++++ assembly/native/build-macos-portable.sh | 4 +--- assembly/native/build-macos-shared.sh | 6 +++--- assembly/native/build-ubuntu-portable.sh | 4 +--- assembly/native/build-ubuntu-shared.sh | 2 +- assembly/nix/build-linux-arm64-nix.sh | 1 + assembly/nix/build-linux-x86-64-nix.sh | 1 + assembly/nix/build-macos-nix.sh | 1 + 9 files changed, 26 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build-ton-linux-android-tonlib.yml b/.github/workflows/build-ton-linux-android-tonlib.yml index bbd956610..a1a6bc99d 100644 --- a/.github/workflows/build-ton-linux-android-tonlib.yml +++ b/.github/workflows/build-ton-linux-android-tonlib.yml @@ -28,5 +28,5 @@ jobs: - name: Upload artifacts uses: actions/upload-artifact@master with: - name: tonlib-android + name: ton-android-tonlib path: artifacts diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 367dd6638..263bd9a43 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -83,6 +83,14 @@ jobs: workflow_conclusion: success skip_unpack: true + - name: Download Android Tonlib artifacts + uses: dawidd6/action-download-artifact@v2 + with: + workflow: build-ton-linux-android-tonlib.yml + path: artifacts + workflow_conclusion: success + skip_unpack: true + - name: Show all artifacts run: | tree artifacts @@ -501,3 +509,11 @@ jobs: file: artifacts/ton-wasm-binaries.zip asset_name: ton-wasm-binaries.zip tag: ${{ steps.tag.outputs.TAG }} + + - name: Upload Android Tonlib artifacts + uses: svenstaro/upload-release-action@v2 + with: + repo_token: ${{ secrets.GITHUB_TOKEN }} + file: artifacts/ton-android-tonlib.zip + asset_name: ton-android-tonlib.zip + tag: ${{ steps.tag.outputs.TAG }} diff --git a/assembly/native/build-macos-portable.sh b/assembly/native/build-macos-portable.sh index b296d3393..d17525f55 100644 --- a/assembly/native/build-macos-portable.sh +++ b/assembly/native/build-macos-portable.sh @@ -192,8 +192,6 @@ if [ "$with_artifacts" = true ]; then echo Creating artifacts... rm -rf artifacts mkdir artifacts - cp crypto/fift/lib artifacts/ - cp -R crypto/smartcont/ artifacts/ cp build/storage/storage-daemon/storage-daemon artifacts/ cp build/storage/storage-daemon/storage-daemon-cli artifacts/ cp build/blockchain-explorer/blockchain-explorer artifacts/ @@ -213,9 +211,9 @@ if [ "$with_artifacts" = true ]; then cp build/utils/json2tlo artifacts/ cp build/adnl/adnl-proxy artifacts/ cp build/emulator/libemulator.dylib artifacts/ - chmod +x artifacts/* rsync -r crypto/smartcont artifacts/ rsync -r crypto/fift/lib artifacts/ + chmod -R +x artifacts/* fi if [ "$with_tests" = true ]; then diff --git a/assembly/native/build-macos-shared.sh b/assembly/native/build-macos-shared.sh index 7574f481a..20cb70cff 100644 --- a/assembly/native/build-macos-shared.sh +++ b/assembly/native/build-macos-shared.sh @@ -140,9 +140,9 @@ if [ "$with_artifacts" = true ]; then cp build/utils/json2tlo artifacts/ cp build/adnl/adnl-proxy artifacts/ cp build/emulator/libemulator.dylib artifacts/ - chmod +x artifacts/* - rsync -r crypto/smartcont artifacts/ - rsync -r crypto/fift/lib artifacts/ + cp -R crypto/smartcont artifacts/ + cp -R crypto/fift/lib artifacts/ + chmod -R +x artifacts/* fi if [ "$with_tests" = true ]; then diff --git a/assembly/native/build-ubuntu-portable.sh b/assembly/native/build-ubuntu-portable.sh index b5a167626..aa2947cb7 100644 --- a/assembly/native/build-ubuntu-portable.sh +++ b/assembly/native/build-ubuntu-portable.sh @@ -193,8 +193,6 @@ cd .. if [ "$with_artifacts" = true ]; then rm -rf artifacts mkdir artifacts - cp crypto/fift/lib artifacts/ - cp -R crypto/smartcont/ artifacts/ mv build/tonlib/libtonlibjson.so.0.5 build/tonlib/libtonlibjson.so cp build/storage/storage-daemon/storage-daemon build/storage/storage-daemon/storage-daemon-cli \ build/crypto/fift build/crypto/tlbc build/crypto/func build/crypto/create-state build/blockchain-explorer/blockchain-explorer \ @@ -204,9 +202,9 @@ if [ "$with_artifacts" = true ]; then build/utils/generate-random-id build/utils/json2tlo build/adnl/adnl-proxy build/emulator/libemulator.so \ artifacts test $? -eq 0 || { echo "Can't copy final binaries"; exit 1; } - chmod +x artifacts/* cp -R crypto/smartcont artifacts cp -R crypto/fift/lib artifacts + chmod -R +x artifacts/* fi if [ "$with_tests" = true ]; then diff --git a/assembly/native/build-ubuntu-shared.sh b/assembly/native/build-ubuntu-shared.sh index 4ce86d81f..a62698d26 100644 --- a/assembly/native/build-ubuntu-shared.sh +++ b/assembly/native/build-ubuntu-shared.sh @@ -112,9 +112,9 @@ if [ "$with_artifacts" = true ]; then build/utils/generate-random-id build/utils/json2tlo build/adnl/adnl-proxy build/emulator/libemulator.so \ artifacts test $? -eq 0 || { echo "Can't copy final binaries"; exit 1; } - chmod +x artifacts/* cp -R crypto/smartcont artifacts cp -R crypto/fift/lib artifacts + chmod -R +x artifacts/* fi if [ "$with_tests" = true ]; then diff --git a/assembly/nix/build-linux-arm64-nix.sh b/assembly/nix/build-linux-arm64-nix.sh index 7e85a8712..6fc2fab20 100644 --- a/assembly/nix/build-linux-arm64-nix.sh +++ b/assembly/nix/build-linux-arm64-nix.sh @@ -35,3 +35,4 @@ cp ./result/lib/libtonlibjson.so.0.5 artifacts/libtonlibjson.so cp ./result/lib/libemulator.so artifacts/ cp ./result/lib/fift/* artifacts/lib/ cp -r ./result/share/ton/smartcont artifacts/ +chmod -R +x artifacts diff --git a/assembly/nix/build-linux-x86-64-nix.sh b/assembly/nix/build-linux-x86-64-nix.sh index c1f1dcf37..30ab79f72 100644 --- a/assembly/nix/build-linux-x86-64-nix.sh +++ b/assembly/nix/build-linux-x86-64-nix.sh @@ -35,3 +35,4 @@ cp ./result/lib/libtonlibjson.so.0.5 artifacts/libtonlibjson.so cp ./result/lib/libemulator.so artifacts/ cp ./result/lib/fift/* artifacts/lib/ cp -r ./result/share/ton/smartcont artifacts/ +chmod -R +x artifacts diff --git a/assembly/nix/build-macos-nix.sh b/assembly/nix/build-macos-nix.sh index c75ca0428..8a07bea20 100644 --- a/assembly/nix/build-macos-nix.sh +++ b/assembly/nix/build-macos-nix.sh @@ -35,3 +35,4 @@ cp ./result/lib/libtonlibjson.dylib artifacts/ cp ./result/lib/libemulator.dylib artifacts/ cp ./result/lib/fift/* artifacts/lib/ cp -r ./result/share/ton/smartcont artifacts/ +chmod -R +x artifacts