From f01605525f448baadf36393c28ea8a014807b78b Mon Sep 17 00:00:00 2001 From: Ruslan Tushov Date: Thu, 28 Nov 2024 10:20:22 +0500 Subject: [PATCH 01/17] rfc 103, candidate core and session (#2282) Signed-off-by: turuslan --- core/network/peer_state.hpp | 2 - .../types/collator_messages_vstaging.hpp | 7 +- .../approval/approval_distribution.cpp | 12 +- .../availability/availability_chunk_index.hpp | 21 +-- .../availability/bitfield/signer.cpp | 4 +- .../availability/recovery/recovery_impl.cpp | 3 +- core/parachain/candidate_descriptor_v2.hpp | 169 ++++++++++++++++++ core/parachain/pvf/pvf_error.hpp | 30 ++++ core/parachain/pvf/pvf_impl.cpp | 39 +++- core/parachain/pvf/pvf_impl.hpp | 19 -- core/parachain/transpose_claim_queue.hpp | 34 ++++ core/parachain/types.hpp | 5 +- core/parachain/ump_signal.hpp | 74 ++++++++ .../validator/impl/parachain_processor.cpp | 37 ++-- .../validator/parachain_processor.hpp | 7 +- .../prospective_parachains/fragment.cpp | 8 +- .../per_relay_parent_state.hpp | 3 + .../statement_distribution.cpp | 43 +++++ .../statement_distribution.hpp | 4 + .../runtime_api/impl/parachain_host.cpp | 12 +- .../runtime_api/impl/parachain_host.hpp | 4 +- core/runtime/runtime_api/node_features.hpp | 56 ++++++ core/runtime/runtime_api/parachain_host.hpp | 34 +--- .../parachain/availability/recovery_test.cpp | 12 +- .../core/parachain/parachain_test_harness.hpp | 4 +- .../core/parachain/prospective_parachains.cpp | 4 +- .../mock/core/runtime/parachain_host_mock.hpp | 4 +- 27 files changed, 513 insertions(+), 138 deletions(-) create mode 100644 core/parachain/candidate_descriptor_v2.hpp create mode 100644 core/parachain/pvf/pvf_error.hpp create mode 100644 core/parachain/transpose_claim_queue.hpp create mode 100644 core/parachain/ump_signal.hpp create mode 100644 core/runtime/runtime_api/node_features.hpp diff --git a/core/network/peer_state.hpp b/core/network/peer_state.hpp index 109d478358..427b731035 100644 --- a/core/network/peer_state.hpp +++ b/core/network/peer_state.hpp @@ -99,7 +99,6 @@ namespace kagome::network { template <> struct std::hash { size_t operator()(const kagome::network::FetchedCollation &value) const { - using CollatorId = kagome::parachain::CollatorId; using CandidateHash = kagome::parachain::CandidateHash; using RelayHash = kagome::parachain::RelayHash; using ParachainId = kagome::parachain::ParachainId; @@ -108,7 +107,6 @@ struct std::hash { boost::hash_combine(result, std::hash()(value.para_id)); boost::hash_combine(result, std::hash()(value.candidate_hash)); - boost::hash_combine(result, std::hash()(value.collator_id)); return result; } diff --git a/core/network/types/collator_messages_vstaging.hpp b/core/network/types/collator_messages_vstaging.hpp index 48ea8582a2..424b5b28da 100644 --- a/core/network/types/collator_messages_vstaging.hpp +++ b/core/network/types/collator_messages_vstaging.hpp @@ -447,16 +447,12 @@ namespace kagome::network { }; struct FetchedCollation { - SCALE_TIE(4); - /// Candidate's relay parent. RelayHash relay_parent; /// Parachain id. ParachainId para_id; /// Candidate hash. CandidateHash candidate_hash; - /// Id of the collator the collation was fetched from. - CollatorId collator_id; static FetchedCollation from(const network::CandidateReceipt &receipt, const crypto::Hasher &hasher) { @@ -465,9 +461,10 @@ namespace kagome::network { .relay_parent = descriptor.relay_parent, .para_id = descriptor.para_id, .candidate_hash = receipt.hash(hasher), - .collator_id = descriptor.collator_id, }; } + + bool operator==(const FetchedCollation &) const = default; }; /** diff --git a/core/parachain/approval/approval_distribution.cpp b/core/parachain/approval/approval_distribution.cpp index 10b9de6562..03f701184b 100644 --- a/core/parachain/approval/approval_distribution.cpp +++ b/core/parachain/approval/approval_distribution.cpp @@ -1115,15 +1115,9 @@ namespace kagome::parachain { } bool enable_v2_assignments = false; - if (auto r = parachain_host_->node_features(block_hash, session_index); - r.has_value()) { - if (r.value() - && r.value()->bits.size() > runtime::ParachainHost::NodeFeatureIndex:: - EnableAssignmentsV2) { - enable_v2_assignments = - r.value()->bits - [runtime::ParachainHost::NodeFeatureIndex::EnableAssignmentsV2]; - } + if (auto r = parachain_host_->node_features(block_hash); r.has_value()) { + enable_v2_assignments = + r.value().has(runtime::NodeFeatures::EnableAssignmentsV2); } approval::UnsafeVRFOutput unsafe_vrf{ diff --git a/core/parachain/availability/availability_chunk_index.hpp b/core/parachain/availability/availability_chunk_index.hpp index d71d9eafaa..0f1deaac79 100644 --- a/core/parachain/availability/availability_chunk_index.hpp +++ b/core/parachain/availability/availability_chunk_index.hpp @@ -44,23 +44,8 @@ namespace kagome::parachain { } inline bool availability_chunk_mapping_is_enabled( - std::optional node_features) { - // none if node_features is not defined - [[unlikely]] if (not node_features.has_value()) { // - return false; - } - - const auto &features = node_features.value(); - - static const auto feature = - runtime::ParachainHost::NodeFeatureIndex::AvailabilityChunkMapping; - - // none if needed feature is out of bound - [[unlikely]] if (feature >= features.bits.size()) { // - return false; - } - - return features.bits[feature]; + const runtime::NodeFeatures &node_features) { + return node_features.has(runtime::NodeFeatures::AvailabilityChunkMapping); } /// Compute the per-validator availability chunk index. @@ -68,7 +53,7 @@ namespace kagome::parachain { /// Any modification to the output of the function needs to be coordinated via /// the runtime. It's best to use minimal/no external dependencies. inline outcome::result availability_chunk_index( - std::optional node_features, + const runtime::NodeFeatures &node_features, size_t n_validators, CoreIndex core_index, ValidatorIndex validator_index) { diff --git a/core/parachain/availability/bitfield/signer.cpp b/core/parachain/availability/bitfield/signer.cpp index f138065aed..640556cc5e 100644 --- a/core/parachain/availability/bitfield/signer.cpp +++ b/core/parachain/availability/bitfield/signer.cpp @@ -79,9 +79,7 @@ namespace kagome::parachain { OUTCOME_TRY( session, parachain_api_->session_info(relay_parent, signer->getSessionIndex())); - OUTCOME_TRY( - node_features, - parachain_api_->node_features(relay_parent, signer->getSessionIndex())); + OUTCOME_TRY(node_features, parachain_api_->node_features(relay_parent)); candidates.reserve(cores.size()); for (CoreIndex core_index = 0; core_index < cores.size(); ++core_index) { auto &core = cores[core_index]; diff --git a/core/parachain/availability/recovery/recovery_impl.cpp b/core/parachain/availability/recovery/recovery_impl.cpp index 4da6689c0e..3f8004a43b 100644 --- a/core/parachain/availability/recovery/recovery_impl.cpp +++ b/core/parachain/availability/recovery/recovery_impl.cpp @@ -134,8 +134,7 @@ namespace kagome::parachain { cb(_min.error()); return; } - auto _node_features = - parachain_api_->node_features(block.hash, session_index); + auto _node_features = parachain_api_->node_features(block.hash); if (_node_features.has_error()) { lock.unlock(); cb(_node_features.error()); diff --git a/core/parachain/candidate_descriptor_v2.hpp b/core/parachain/candidate_descriptor_v2.hpp new file mode 100644 index 0000000000..a728adb1d9 --- /dev/null +++ b/core/parachain/candidate_descriptor_v2.hpp @@ -0,0 +1,169 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include "crypto/sr25519_provider.hpp" +#include "parachain/pvf/pvf_error.hpp" +#include "parachain/transpose_claim_queue.hpp" +#include "parachain/types.hpp" +#include "parachain/ump_signal.hpp" + +namespace kagome::network { + // https://github.com/paritytech/polkadot-sdk/blob/1e3b8e1639c1cf784eabf0a9afcab1f3987e0ca4/polkadot/primitives/src/vstaging/mod.rs#L44 + /// The default claim queue offset to be used if it's not + /// configured/accessible in the parachain + /// runtime + constexpr uint8_t kDefaultClaimQueueOffset = 0; + + // https://github.com/paritytech/polkadot-sdk/blob/1e3b8e1639c1cf784eabf0a9afcab1f3987e0ca4/polkadot/primitives/src/vstaging/mod.rs#L532-L534 + inline bool isV1(const CandidateDescriptor &descriptor) { + constexpr auto is_zero = [](BufferView xs) { + return static_cast(std::ranges::count(xs, 0)) == xs.size(); + }; + return not is_zero(std::span{descriptor.reserved_1}.subspan(7)) + or not is_zero(descriptor.reserved_2); + } + + // https://github.com/paritytech/polkadot-sdk/blob/1e3b8e1639c1cf784eabf0a9afcab1f3987e0ca4/polkadot/primitives/src/vstaging/mod.rs#L532-L537 + inline bool isV2(const CandidateDescriptor &descriptor) { + return not isV1(descriptor) and descriptor.reserved_1[0] == 0; + } + + // https://github.com/paritytech/polkadot-sdk/blob/1e3b8e1639c1cf784eabf0a9afcab1f3987e0ca4/polkadot/primitives/src/vstaging/mod.rs#L179 + /// Check the signature of the collator within this descriptor. + inline outcome::result checkSignature( + const crypto::Sr25519Provider &sr25519, + const CandidateDescriptor &descriptor) { + if (not isV1(descriptor)) { + return outcome::success(); + } + OUTCOME_TRY( + r, + sr25519.verify(crypto::Sr25519Signature{descriptor.reserved_2}, + descriptor.signable(), + crypto::Sr25519PublicKey{descriptor.reserved_1})); + if (not r) { + return PvfError::SIGNATURE; + } + return outcome::success(); + } + + // https://github.com/paritytech/polkadot-sdk/blob/1e3b8e1639c1cf784eabf0a9afcab1f3987e0ca4/polkadot/primitives/src/vstaging/mod.rs#L580 + /// Returns the `core_index` of `V2` candidate descriptors, `None` otherwise. + inline std::optional coreIndex( + const CandidateDescriptor &descriptor) { + if (isV1(descriptor)) { + return std::nullopt; + } + return boost::endian::load_little_u16(&descriptor.reserved_1[1]); + } + + // https://github.com/paritytech/polkadot-sdk/blob/1e3b8e1639c1cf784eabf0a9afcab1f3987e0ca4/polkadot/primitives/src/vstaging/mod.rs#L589 + /// Returns the `core_index` of `V2` candidate descriptors, `None` otherwise. + inline std::optional sessionIndex( + const CandidateDescriptor &descriptor) { + if (isV1(descriptor)) { + return std::nullopt; + } + return boost::endian::load_little_u32(&descriptor.reserved_1[3]); + } + + enum class CheckCoreIndexError : uint8_t { + InvalidCoreIndex, + NoAssignment, + UnknownVersion, + InvalidSession, + }; + Q_ENUM_ERROR_CODE(CheckCoreIndexError) { + using E = decltype(e); + switch (e) { + case E::InvalidCoreIndex: + return "The specified core index is invalid"; + case E::NoAssignment: + return "The parachain is not assigned to any core at specified claim " + "queue offset"; + case E::UnknownVersion: + return "Unknown internal version"; + case E::InvalidSession: + return "Invalid session"; + } + abort(); + } + + // https://github.com/paritytech/polkadot-sdk/blob/1e3b8e1639c1cf784eabf0a9afcab1f3987e0ca4/polkadot/primitives/src/vstaging/mod.rs#L602 + /// Checks if descriptor core index is equal to the committed core index. + /// Input `cores_per_para` is a claim queue snapshot at the candidate's relay + /// parent, stored as a mapping between `ParaId` and the cores assigned per + /// depth. + inline outcome::result checkCoreIndex( + const CommittedCandidateReceipt &receipt, + const TransposedClaimQueue &claims) { + if (isV1(receipt.descriptor)) { + return outcome::success(); + } + if (not isV2(receipt.descriptor)) { + return CheckCoreIndexError::UnknownVersion; + } + OUTCOME_TRY(selector, coreSelector(receipt.commitments)); + auto offset = + selector ? selector->claim_queue_offset : kDefaultClaimQueueOffset; + auto it1 = claims.find(receipt.descriptor.para_id); + if (it1 == claims.end()) { + return CheckCoreIndexError::NoAssignment; + } + auto it2 = it1->second.find(offset); + if (it2 == it1->second.end()) { + return CheckCoreIndexError::NoAssignment; + } + auto &assigned_cores = it2->second; + if (assigned_cores.empty()) { + return CheckCoreIndexError::NoAssignment; + } + auto core = coreIndex(receipt.descriptor).value(); + if (not selector and assigned_cores.size() > 1) { + if (not assigned_cores.contains(core)) { + return CheckCoreIndexError::InvalidCoreIndex; + } + return outcome::success(); + } + auto expected_core = + *std::next(assigned_cores.begin(), + selector ? static_cast(selector->core_selector + % assigned_cores.size()) + : 0); + if (core != expected_core) { + return CheckCoreIndexError::InvalidCoreIndex; + } + return outcome::success(); + } + + // https://github.com/paritytech/polkadot-sdk/blob/1e3b8e1639c1cf784eabf0a9afcab1f3987e0ca4/polkadot/node/network/collator-protocol/src/validator_side/mod.rs#L2114 + inline outcome::result descriptorVersionSanityCheck( + const CandidateDescriptor &descriptor, + bool v2_receipts, + CoreIndex expected_core, + SessionIndex expected_session) { + if (isV1(descriptor)) { + return outcome::success(); + } + if (not isV2(descriptor)) { + return CheckCoreIndexError::UnknownVersion; + } + if (not v2_receipts) { + return CheckCoreIndexError::UnknownVersion; + } + if (coreIndex(descriptor) != expected_core) { + return CheckCoreIndexError::InvalidCoreIndex; + } + if (sessionIndex(descriptor) != expected_session) { + return CheckCoreIndexError::InvalidSession; + } + return outcome::success(); + } +} // namespace kagome::network diff --git a/core/parachain/pvf/pvf_error.hpp b/core/parachain/pvf/pvf_error.hpp new file mode 100644 index 0000000000..0cad353ae8 --- /dev/null +++ b/core/parachain/pvf/pvf_error.hpp @@ -0,0 +1,30 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include + +namespace kagome::parachain { + enum class PvfError : uint8_t { + // NO_DATA conflicted with + NO_PERSISTED_DATA = 1, + POV_SIZE, + POV_HASH, + CODE_HASH, + SIGNATURE, + HEAD_HASH, + COMMITMENTS_HASH, + OUTPUTS, + PERSISTED_DATA_HASH, + NO_CODE, + COMPILATION_ERROR, + }; +} // namespace kagome::parachain + +OUTCOME_HPP_DECLARE_ERROR(kagome::parachain, PvfError) diff --git a/core/parachain/pvf/pvf_impl.cpp b/core/parachain/pvf/pvf_impl.cpp index 06d9bad961..327a8128c6 100644 --- a/core/parachain/pvf/pvf_impl.cpp +++ b/core/parachain/pvf/pvf_impl.cpp @@ -14,8 +14,10 @@ #include "common/visitor.hpp" #include "log/profiling_logger.hpp" #include "metrics/histogram_timer.hpp" +#include "parachain/candidate_descriptor_v2.hpp" #include "parachain/pvf/module_precompiler.hpp" #include "parachain/pvf/pool.hpp" +#include "parachain/pvf/pvf_error.hpp" #include "parachain/pvf/pvf_thread_pool.hpp" #include "parachain/pvf/pvf_worker_types.hpp" #include "parachain/pvf/session_params.hpp" @@ -222,6 +224,18 @@ namespace kagome::parachain { code_zstd, timeout_kind, std::move(cb)); + // https://github.com/paritytech/polkadot-sdk/blob/1e3b8e1639c1cf784eabf0a9afcab1f3987e0ca4/polkadot/node/core/candidate-validation/src/lib.rs#L763-L782 + auto session = sessionIndex(receipt.descriptor); + if (session and timeout_kind == runtime::PvfExecTimeoutKind::Backing) { + CB_TRY(auto expected_session, + parachain_api_->session_index_for_child( + receipt.descriptor.relay_parent)); + if (sessionIndex(receipt.descriptor) != expected_session) { + cb(network::CheckCoreIndexError::InvalidSession); + return; + } + } + CB_TRY(auto pov_encoded, scale::encode(pov)); if (pov_encoded.size() > data.max_pov_size) { return cb(PvfError::POV_SIZE); @@ -234,13 +248,7 @@ namespace kagome::parachain { if (code_hash != receipt.descriptor.validation_code_hash) { return cb(PvfError::CODE_HASH); } - CB_TRY(auto signature_valid, - sr25519_provider_->verify(receipt.descriptor.signature, - receipt.descriptor.signable(), - receipt.descriptor.collator_id)); - if (!signature_valid) { - return cb(PvfError::SIGNATURE); - } + CB_TRYV(checkSignature(*sr25519_provider_, receipt.descriptor)); auto timer = metric_pvf_execution_time.timer(); ValidationParams params; @@ -257,6 +265,7 @@ namespace kagome::parachain { libp2p::SharedFn{[weak_self{weak_from_this()}, data, receipt, + timeout_kind, cb{std::move(cb)}, timer{std::move(timer)}]( outcome::result r) { @@ -267,6 +276,22 @@ namespace kagome::parachain { CB_TRY(auto result, std::move(r)); CB_TRY(auto commitments, self->fromOutputs(receipt, std::move(result))); + // https://github.com/paritytech/polkadot-sdk/blob/1e3b8e1639c1cf784eabf0a9afcab1f3987e0ca4/polkadot/node/core/candidate-validation/src/lib.rs#L915-L951 + if (timeout_kind == runtime::PvfExecTimeoutKind::Backing + and coreIndex(receipt.descriptor)) { + CB_TRY(auto claims, + self->parachain_api_->claim_queue( + receipt.descriptor.relay_parent)); + if (not claims) { + claims.emplace(); + } + CB_TRYV(network::checkCoreIndex( + { + .descriptor = receipt.descriptor, + .commitments = commitments, + }, + transposeClaimQueue(*claims))); + } cb(std::make_pair(std::move(commitments), data)); }}); } diff --git a/core/parachain/pvf/pvf_impl.hpp b/core/parachain/pvf/pvf_impl.hpp index 47c2aac7e8..24af532134 100644 --- a/core/parachain/pvf/pvf_impl.hpp +++ b/core/parachain/pvf/pvf_impl.hpp @@ -34,25 +34,6 @@ namespace kagome::runtime { class RuntimeInstancesPool; } // namespace kagome::runtime -namespace kagome::parachain { - enum class PvfError { - // NO_DATA conflicted with - NO_PERSISTED_DATA = 1, - POV_SIZE, - POV_HASH, - CODE_HASH, - SIGNATURE, - HEAD_HASH, - COMMITMENTS_HASH, - OUTPUTS, - PERSISTED_DATA_HASH, - NO_CODE, - COMPILATION_ERROR - }; -} // namespace kagome::parachain - -OUTCOME_HPP_DECLARE_ERROR(kagome::parachain, PvfError) - namespace kagome::parachain { class PvfPool; class PvfThreadPool; diff --git a/core/parachain/transpose_claim_queue.hpp b/core/parachain/transpose_claim_queue.hpp new file mode 100644 index 0000000000..9e487f7b8c --- /dev/null +++ b/core/parachain/transpose_claim_queue.hpp @@ -0,0 +1,34 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include + +#include "runtime/runtime_api/parachain_host_types.hpp" + +namespace kagome::parachain { + /// The claim queue mapped by parachain id. + using TransposedClaimQueue = + std::map>>; + + /// Returns a mapping between the para id and the core indices assigned at + /// different + /// depths in the claim queue. + inline TransposedClaimQueue transposeClaimQueue( + const runtime::ClaimQueueSnapshot &claims) { + TransposedClaimQueue r; + for (auto &[core, paras] : claims.claimes) { + size_t depth = 0; + for (auto ¶ : paras) { + r[para][depth].emplace(core); + ++depth; + } + } + return r; + } +} // namespace kagome::parachain diff --git a/core/parachain/types.hpp b/core/parachain/types.hpp index eb7e7cb530..572778dda6 100644 --- a/core/parachain/types.hpp +++ b/core/parachain/types.hpp @@ -156,15 +156,14 @@ namespace kagome::network { primitives::BlockHash relay_parent; /// Hash of the relay chain block the candidate is /// executed in the context of - parachain::CollatorPublicKey collator_id; /// Collators public key. + common::Blob<32> reserved_1; primitives::BlockHash persisted_data_hash; /// Hash of the persisted validation data primitives::BlockHash pov_hash; /// Hash of the PoV block. storage::trie::RootHash erasure_encoding_root; /// Root of the block’s erasure encoding Merkle /// tree. - parachain::Signature - signature; /// Collator signature of the concatenated components + common::Blob<64> reserved_2; primitives::BlockHash para_head_hash; /// Hash of the parachain head data of this candidate. primitives::BlockHash diff --git a/core/parachain/ump_signal.hpp b/core/parachain/ump_signal.hpp new file mode 100644 index 0000000000..a5a0cfda7d --- /dev/null +++ b/core/parachain/ump_signal.hpp @@ -0,0 +1,74 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include + +#include "parachain/types.hpp" + +namespace kagome::parachain { + // https://github.com/paritytech/polkadot-sdk/blob/1e3b8e1639c1cf784eabf0a9afcab1f3987e0ca4/polkadot/primitives/src/vstaging/mod.rs#L435 + /// Separator between `XCM` and `UMPSignal`. + inline const Buffer kUmpSeparator; + + enum class UmpError : uint8_t { + TooManyUMPSignals, + }; + Q_ENUM_ERROR_CODE(UmpError) { + using E = decltype(e); + switch (e) { + case E::TooManyUMPSignals: + return "Too many UMP signals"; + } + abort(); + } + + // https://github.com/paritytech/polkadot-sdk/blob/1e3b8e1639c1cf784eabf0a9afcab1f3987e0ca4/polkadot/primitives/src/vstaging/mod.rs#L432 + /// A message sent by a parachain to select the core the candidate is + /// committed to. Relay chain validators, in particular backers, use the + /// `CoreSelector` and `ClaimQueueOffset` to compute the index of the core the + /// candidate has committed to. + struct UMPSignalSelectCore { + SCALE_TIE(2); + + uint8_t core_selector; + uint8_t claim_queue_offset; + }; + // https://github.com/paritytech/polkadot-sdk/blob/1e3b8e1639c1cf784eabf0a9afcab1f3987e0ca4/polkadot/primitives/src/vstaging/mod.rs#L428 + /// Signals that a parachain can send to the relay chain via the UMP queue. + using UMPSignal = boost::variant; + + // https://github.com/paritytech/polkadot-sdk/blob/1e3b8e1639c1cf784eabf0a9afcab1f3987e0ca4/polkadot/primitives/src/vstaging/mod.rs#L438 + /// Utility function for skipping the ump signals. + inline auto skipUmpSignals(std::span messages) { + return messages.first(std::ranges::find(messages, kUmpSeparator) + - messages.begin()); + } + + // https://github.com/paritytech/polkadot-sdk/blob/1e3b8e1639c1cf784eabf0a9afcab1f3987e0ca4/polkadot/primitives/src/vstaging/mod.rs#L447 + /// Returns the core selector and claim queue offset determined by + /// `UMPSignal::SelectCore` commitment, if present. + inline outcome::result> coreSelector( + const network::CandidateCommitments &commitments) { + auto it = std::ranges::find(commitments.upward_msgs, kUmpSeparator); + if (it == commitments.upward_msgs.end()) { + return std::nullopt; + } + ++it; + if (it == commitments.upward_msgs.end()) { + return std::nullopt; + } + OUTCOME_TRY(signal, scale::decode(*it)); + ++it; + if (it != commitments.upward_msgs.end()) { + return UmpError::TooManyUMPSignals; + } + return boost::get(signal); + } +} // namespace kagome::parachain diff --git a/core/parachain/validator/impl/parachain_processor.cpp b/core/parachain/validator/impl/parachain_processor.cpp index f272db5419..eb819d6e6f 100644 --- a/core/parachain/validator/impl/parachain_processor.cpp +++ b/core/parachain/validator/impl/parachain_processor.cpp @@ -28,6 +28,7 @@ #include "network/router.hpp" #include "parachain/availability/chunks.hpp" #include "parachain/availability/proof.hpp" +#include "parachain/candidate_descriptor_v2.hpp" #include "parachain/candidate_view.hpp" #include "parachain/peer_relay_parent_knowledge.hpp" #include "scale/scale.hpp" @@ -468,17 +469,9 @@ namespace kagome::parachain { return Error::NO_SESSION_INFO; } - bool inject_core_index = false; - if (auto r = parachain_host_->node_features(relay_parent, session_index); - r.has_value()) { - if (r.value() - && r.value()->bits.size() > runtime::ParachainHost::NodeFeatureIndex:: - ElasticScalingMVP) { - inject_core_index = - r.value()->bits - [runtime::ParachainHost::NodeFeatureIndex::ElasticScalingMVP]; - } - } + OUTCOME_TRY(node_features, parachain_host_->node_features(relay_parent)); + auto inject_core_index = + node_features.has(runtime::NodeFeatures::ElasticScalingMVP); uint32_t minimum_backing_votes = 2; /// legacy value if (auto r = @@ -493,8 +486,20 @@ namespace kagome::parachain { } std::optional validator_index; + // https://github.com/paritytech/polkadot-sdk/blob/1e3b8e1639c1cf784eabf0a9afcab1f3987e0ca4/polkadot/node/network/collator-protocol/src/validator_side/mod.rs#L487-L495 + CoreIndex current_core = 0; if (validator) { validator_index = validator->validatorIndex(); + + size_t i_group = 0; + for (auto &group : validator_groups) { + if (group.contains(validator->validatorIndex())) { + current_core = + group_rotation_info.coreForGroup(i_group, cores.size()); + break; + } + ++i_group; + } } OUTCOME_TRY(global_v_index, @@ -592,6 +597,9 @@ namespace kagome::parachain { .fallbacks = {}, .backed_hashes = {}, .inject_core_index = inject_core_index, + .v2_receipts = + node_features.has(runtime::NodeFeatures::CandidateReceiptV2), + .current_core = current_core, .per_session_state = per_session_state, }; } @@ -2679,6 +2687,13 @@ namespace kagome::parachain { auto relay_parent = pending_collation.relay_parent; OUTCOME_TRY(per_relay_parent, getStateByRelayParent(relay_parent)); + + OUTCOME_TRY(descriptorVersionSanityCheck( + pending_collation_fetch.candidate_receipt.descriptor, + per_relay_parent.get().v2_receipts, + per_relay_parent.get().current_core, + per_relay_parent.get().per_session_state->value().session)); + auto &collations = per_relay_parent.get().collations; auto fetched_collation = network::FetchedCollation::from( pending_collation_fetch.candidate_receipt, *hasher_); diff --git a/core/parachain/validator/parachain_processor.hpp b/core/parachain/validator/parachain_processor.hpp index 29a50db570..3a15e5ba24 100644 --- a/core/parachain/validator/parachain_processor.hpp +++ b/core/parachain/validator/parachain_processor.hpp @@ -372,6 +372,8 @@ namespace kagome::parachain { std::unordered_set backed_hashes; bool inject_core_index; + bool v2_receipts; + CoreIndex current_core; std::shared_ptr::RefObj> per_session_state; }; @@ -639,11 +641,6 @@ namespace kagome::parachain { }); } - const network::CollatorPublicKey &collatorIdFromDescriptor( - const network::CandidateDescriptor &descriptor) { - return descriptor.collator_id; - } - /* * Notification */ diff --git a/core/parachain/validator/prospective_parachains/fragment.cpp b/core/parachain/validator/prospective_parachains/fragment.cpp index c04f4d727b..9bdabba5d0 100644 --- a/core/parachain/validator/prospective_parachains/fragment.cpp +++ b/core/parachain/validator/prospective_parachains/fragment.cpp @@ -5,6 +5,8 @@ */ #include "parachain/validator/prospective_parachains/fragment.hpp" + +#include "parachain/ump_signal.hpp" #include "utils/stringify.hpp" #define COMPONENT Fragment @@ -142,7 +144,7 @@ namespace kagome::parachain::fragment { } uint32_t ump_sent_bytes = 0ull; - for (const auto &m : commitments.upward_msgs) { + for (const auto &m : skipUmpSignals(commitments.upward_msgs)) { ump_sent_bytes += uint32_t(m.size()); } @@ -150,9 +152,9 @@ namespace kagome::parachain::fragment { .required_parent = commitments.para_head, .hrmp_watermark = ((commitments.watermark == relay_parent.number) ? HrmpWatermarkUpdate{HrmpWatermarkUpdateHead{ - .v = commitments.watermark}} + .v = commitments.watermark}} : HrmpWatermarkUpdate{HrmpWatermarkUpdateTrunk{ - .v = commitments.watermark}}), + .v = commitments.watermark}}), .outbound_hrmp = outbound_hrmp, .ump_messages_sent = uint32_t(commitments.upward_msgs.size()), .ump_bytes_sent = ump_sent_bytes, diff --git a/core/parachain/validator/statement_distribution/per_relay_parent_state.hpp b/core/parachain/validator/statement_distribution/per_relay_parent_state.hpp index cfe90c8480..78106c7383 100644 --- a/core/parachain/validator/statement_distribution/per_relay_parent_state.hpp +++ b/core/parachain/validator/statement_distribution/per_relay_parent_state.hpp @@ -14,6 +14,7 @@ #include "common/ref_cache.hpp" #include "parachain/backing/cluster.hpp" #include "parachain/backing/grid_tracker.hpp" +#include "parachain/transpose_claim_queue.hpp" #include "parachain/types.hpp" #include "parachain/validator/impl/statements_store.hpp" #include "parachain/validator/statement_distribution/per_session_state.hpp" @@ -47,6 +48,8 @@ namespace kagome::parachain::statement_distribution { SessionIndex session; std::unordered_map> groups_per_para; std::unordered_set disabled_validators; + bool v2_receipts; + TransposedClaimQueue transposed_claim_queue; std::shared_ptr::RefObj> per_session_state; diff --git a/core/parachain/validator/statement_distribution/statement_distribution.cpp b/core/parachain/validator/statement_distribution/statement_distribution.cpp index c177bb9829..52d25e589c 100644 --- a/core/parachain/validator/statement_distribution/statement_distribution.cpp +++ b/core/parachain/validator/statement_distribution/statement_distribution.cpp @@ -8,6 +8,7 @@ #include "network/impl/protocols/fetch_attested_candidate.hpp" #include "network/impl/protocols/parachain.hpp" +#include "parachain/candidate_descriptor_v2.hpp" #include "parachain/validator/parachain_processor.hpp" #include "utils/weak_macro.hpp" @@ -427,6 +428,11 @@ namespace kagome::parachain::statement_distribution { const auto &[_, group_rotation_info] = groups; OUTCOME_TRY(maybe_claim_queue, parachain_host->claim_queue(relay_parent)); + TransposedClaimQueue transposed_claim_queue; + if (maybe_claim_queue) { + transposed_claim_queue = transposeClaimQueue(*maybe_claim_queue); + } + OUTCOME_TRY(node_features, parachain_host->node_features(relay_parent)); auto local_validator = [&]() -> std::optional { if (!per_session_state.value()->value().v_index) { @@ -460,6 +466,9 @@ namespace kagome::parachain::statement_distribution { .session = session_index, .groups_per_para = std::move(groups_per_para), .disabled_validators = std::move(disabled_validators), + .v2_receipts = + node_features.has(runtime::NodeFeatures::CandidateReceiptV2), + .transposed_claim_queue = transposed_claim_queue, .per_session_state = per_session_state.value(), }); } // for @@ -1005,6 +1014,31 @@ namespace kagome::parachain::statement_distribution { }); } + bool StatementDistribution::validate( + const PerRelayParentState &per_relay_parent, + const CandidateHash &candidate_hash, + const network::vstaging::AttestedCandidateResponse &response) const { + if (candidateHash(*hasher, response.candidate_receipt) != candidate_hash) { + return false; + } + // https://github.com/paritytech/polkadot-sdk/blob/1e3b8e1639c1cf784eabf0a9afcab1f3987e0ca4/polkadot/node/network/statement-distribution/src/v2/requests.rs#L744-L772 + if (not per_relay_parent.v2_receipts + and isV2(response.candidate_receipt.descriptor)) { + return false; + } + if (auto r = checkCoreIndex(response.candidate_receipt, + per_relay_parent.transposed_claim_queue); + not r) { + return false; + } + if (auto session = sessionIndex(response.candidate_receipt.descriptor)) { + if (session != per_relay_parent.session) { + return false; + } + } + return true; + } + void StatementDistribution::handle_response( outcome::result &&r, const RelayHash &relay_parent, @@ -1043,6 +1077,15 @@ namespace kagome::parachain::statement_distribution { candidate_hash, group_index, response.statements.size()); + + if (not validate(parachain_state->get(), candidate_hash, response)) { + SL_WARN(logger, + "Invalid response receipt relay_parent={} candidate={}", + relay_parent, + candidate_hash); + return; + } + for (const auto &statement : response.statements) { parachain_state->get().statement_store.insert( parachain_state->get().per_session_state->value().groups, diff --git a/core/parachain/validator/statement_distribution/statement_distribution.hpp b/core/parachain/validator/statement_distribution/statement_distribution.hpp index cad21d2bec..29d3e442d3 100644 --- a/core/parachain/validator/statement_distribution/statement_distribution.hpp +++ b/core/parachain/validator/statement_distribution/statement_distribution.hpp @@ -148,6 +148,10 @@ namespace kagome::parachain::statement_distribution { outcome::result> getStateByRelayParent(const primitives::BlockHash &relay_parent); + bool validate( + const PerRelayParentState &per_relay_parent, + const CandidateHash &candidate_hash, + const network::vstaging::AttestedCandidateResponse &response) const; void handle_response( outcome::result &&r, const RelayHash &relay_parent, diff --git a/core/runtime/runtime_api/impl/parachain_host.cpp b/core/runtime/runtime_api/impl/parachain_host.cpp index 03839557f3..a69fb90771 100644 --- a/core/runtime/runtime_api/impl/parachain_host.cpp +++ b/core/runtime/runtime_api/impl/parachain_host.cpp @@ -300,12 +300,12 @@ namespace kagome::runtime { ctx, "ParachainHost_disabled_validators")); } - outcome::result> - ParachainHostImpl::node_features(const primitives::BlockHash &block, - SessionIndex index) { + outcome::result ParachainHostImpl::node_features( + const primitives::BlockHash &block) { OUTCOME_TRY(ctx, executor_->ctx().ephemeralAt(block)); - return ifExport(executor_->call( - ctx, "ParachainHost_node_features")); + OUTCOME_TRY(r, + ifExport(executor_->call( + ctx, "ParachainHost_node_features"))); + return NodeFeatures{std::move(r)}; } - } // namespace kagome::runtime diff --git a/core/runtime/runtime_api/impl/parachain_host.hpp b/core/runtime/runtime_api/impl/parachain_host.hpp index 2d96568bc2..a00b904e7b 100644 --- a/core/runtime/runtime_api/impl/parachain_host.hpp +++ b/core/runtime/runtime_api/impl/parachain_host.hpp @@ -115,8 +115,8 @@ namespace kagome::runtime { outcome::result> disabled_validators( const primitives::BlockHash &block) override; - outcome::result> node_features( - const primitives::BlockHash &block, SessionIndex index) override; + outcome::result node_features( + const primitives::BlockHash &block) override; ClaimQueueResult claim_queue(const primitives::BlockHash &block) override; diff --git a/core/runtime/runtime_api/node_features.hpp b/core/runtime/runtime_api/node_features.hpp new file mode 100644 index 0000000000..cae8d1156f --- /dev/null +++ b/core/runtime/runtime_api/node_features.hpp @@ -0,0 +1,56 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include + +#include + +namespace kagome::runtime { + struct NodeFeatures { + /// A feature index used to identify a bit into the node_features array + /// stored in the HostConfiguration. + enum Index : uint8_t { + /// Tells if tranch0 assignments could be sent in a single certificate. + /// Reserved for: + /// `` + EnableAssignmentsV2 = 0, + + /// This feature enables the extension of + /// `BackedCandidate::validator_indices` by 8 bits. + /// The value stored there represents the assumed core index where the + /// candidates are backed. This is needed for the elastic scaling MVP. + ElasticScalingMVP = 1, + + /// Tells if the chunk mapping feature is enabled. + /// Enables the implementation of + /// [RFC-47](https://github.com/polkadot-fellows/RFCs/blob/main/text/0047-assignment-of-availability-chunks.md). + /// Must not be enabled unless all validators and collators have stopped + /// using `req_chunk` protocol version 1. If it is enabled, validators can + /// start systematic chunk recovery. + AvailabilityChunkMapping = 2, + + /// Enables node side support of `CoreIndex` committed candidate receipts. + /// See [RFC-103](https://github.com/polkadot-fellows/RFCs/pull/103) for + /// details. + /// Only enable if at least 2/3 of nodes support the feature. + CandidateReceiptV2 = 3, + + /// First unassigned feature bit. + /// Every time a new feature flag is assigned it should take this value. + /// and this should be incremented. + FirstUnassigned = 4, + }; + + bool has(Index index) const { + return bits and index < bits->bits.size() && bits->bits.at(index); + } + + std::optional bits; + }; +} // namespace kagome::runtime diff --git a/core/runtime/runtime_api/parachain_host.hpp b/core/runtime/runtime_api/parachain_host.hpp index f93aac75d0..c008d10bbd 100644 --- a/core/runtime/runtime_api/parachain_host.hpp +++ b/core/runtime/runtime_api/parachain_host.hpp @@ -14,6 +14,7 @@ #include "primitives/block_id.hpp" #include "primitives/common.hpp" #include "primitives/parachain_host.hpp" +#include "runtime/runtime_api/node_features.hpp" #include "runtime/runtime_api/parachain_host_types.hpp" namespace kagome::runtime { @@ -23,35 +24,6 @@ namespace kagome::runtime { public: virtual ~ParachainHost() = default; - using NodeFeatures = scale::BitVec; - /// A feature index used to identify a bit into the node_features array - /// stored in the HostConfiguration. - enum NodeFeatureIndex { - /// Tells if tranch0 assignments could be sent in a single certificate. - /// Reserved for: - /// `` - EnableAssignmentsV2 = 0, - - /// This feature enables the extension of - /// `BackedCandidate::validator_indices` by 8 bits. - /// The value stored there represents the assumed core index where the - /// candidates are backed. This is needed for the elastic scaling MVP. - ElasticScalingMVP = 1, - - /// Tells if the chunk mapping feature is enabled. - /// Enables the implementation of - /// [RFC-47](https://github.com/polkadot-fellows/RFCs/blob/main/text/0047-assignment-of-availability-chunks.md). - /// Must not be enabled unless all validators and collators have stopped - /// using `req_chunk` protocol version 1. If it is enabled, validators can - /// start systematic chunk recovery. - AvailabilityChunkMapping = 2, - - /// First unassigned feature bit. - /// Every time a new feature flag is assigned it should take this value. - /// and this should be incremented. - FirstUnassigned = 3, - }; - /** * @brief Calls the ParachainHost_active_parachains function from wasm code * @return vector of ParachainId items or error if fails @@ -255,8 +227,8 @@ namespace kagome::runtime { virtual outcome::result> disabled_validators( const primitives::BlockHash &block) = 0; - virtual outcome::result> node_features( - const primitives::BlockHash &block, SessionIndex index) = 0; + virtual outcome::result node_features( + const primitives::BlockHash &block) = 0; using ClaimQueueResult = outcome::result>; virtual ClaimQueueResult claim_queue( diff --git a/test/core/parachain/availability/recovery_test.cpp b/test/core/parachain/availability/recovery_test.cpp index a3cfacf10f..5c460a3fd9 100644 --- a/test/core/parachain/availability/recovery_test.cpp +++ b/test/core/parachain/availability/recovery_test.cpp @@ -46,6 +46,7 @@ using kagome::parachain::ValidatorId; using kagome::primitives::AuthorityDiscoveryId; using kagome::primitives::BlockInfo; using kagome::runtime::AvailableData; +using kagome::runtime::NodeFeatures; using kagome::runtime::ParachainHost; using kagome::runtime::ParachainHostMock; using kagome::runtime::SessionInfo; @@ -92,13 +93,12 @@ class RecoveryTest : public testing::Test { parachain_api = std::make_shared(); ON_CALL(*parachain_api, session_info(best_block.hash, session_index)) .WillByDefault(Invoke([&] { return session; })); - ON_CALL(*parachain_api, node_features(best_block.hash, session_index)) + ON_CALL(*parachain_api, node_features(best_block.hash)) .WillByDefault(Invoke([&] { - scale::BitVec nf{}; - nf.bits.resize(ParachainHost::NodeFeatureIndex::FirstUnassigned); - nf.bits[ParachainHost::NodeFeatureIndex::AvailabilityChunkMapping] = - true; - return std::optional{nf}; + scale::BitVec bits; + bits.bits.resize(NodeFeatures::FirstUnassigned); + bits.bits[NodeFeatures::AvailabilityChunkMapping] = true; + return NodeFeatures{.bits = std::move(bits)}; })); av_store = std::make_shared(); diff --git a/test/core/parachain/parachain_test_harness.hpp b/test/core/parachain/parachain_test_harness.hpp index f28745b5ab..7957936b46 100644 --- a/test/core/parachain/parachain_test_harness.hpp +++ b/test/core/parachain/parachain_test_harness.hpp @@ -121,11 +121,11 @@ class ProspectiveParachainsTestHarness : public testing::Test { network::CandidateDescriptor{ .para_id = para_id, .relay_parent = relay_parent, - .collator_id = {}, + .reserved_1 = {}, .persisted_data_hash = persisted_validation_data.getHash(), .pov_hash = fromNumber(1), .erasure_encoding_root = fromNumber(1), - .signature = {}, + .reserved_2 = {}, .para_head_hash = hasher_->blake2b_256(para_head), .validation_code_hash = fromNumber(42), }, diff --git a/test/core/parachain/prospective_parachains.cpp b/test/core/parachain/prospective_parachains.cpp index 5219de061a..c3d74c173c 100644 --- a/test/core/parachain/prospective_parachains.cpp +++ b/test/core/parachain/prospective_parachains.cpp @@ -348,11 +348,11 @@ class ProspectiveParachainsTest : public ProspectiveParachainsTestHarness { return network::CandidateDescriptor{ .para_id = 0, .relay_parent = relay_parent, - .collator_id = {}, + .reserved_1 = {}, .persisted_data_hash = fromNumber(0), .pov_hash = fromNumber(0), .erasure_encoding_root = fromNumber(0), - .signature = {}, + .reserved_2 = {}, .para_head_hash = fromNumber(0), .validation_code_hash = crypto::Hashed>, + MOCK_METHOD(outcome::result, node_features, - (const primitives::BlockHash &, SessionIndex), + (const primitives::BlockHash &), (override)); MOCK_METHOD(ClaimQueueResult, From 778d2140bb4f7494b53ad78ec323c465064df7c2 Mon Sep 17 00:00:00 2001 From: Boris <31869190+ErakhtinB@users.noreply.github.com> Date: Thu, 28 Nov 2024 13:21:17 +0500 Subject: [PATCH 02/17] Sys info metrics (#2294) --- core/telemetry/impl/connection_impl.cpp | 9 +- core/telemetry/impl/service_impl.cpp | 194 +++++++++++++++++++++++- 2 files changed, 200 insertions(+), 3 deletions(-) diff --git a/core/telemetry/impl/connection_impl.cpp b/core/telemetry/impl/connection_impl.cpp index 24e62e3388..5c3a495f53 100644 --- a/core/telemetry/impl/connection_impl.cpp +++ b/core/telemetry/impl/connection_impl.cpp @@ -84,6 +84,11 @@ namespace kagome::telemetry { auto &path = endpoint_.uri().Path; path_ = path.empty() ? "/" : path; + // https://github.com/qdrvm/kagome/discussions/2265#discussioncomment-11245398 + if (path_.back() != '/') { + path_ += '/'; + } + if (secure_) { if (not ssl_ctx_) { ssl_ctx_.emplace(endpoint_.uri().Host); @@ -153,9 +158,9 @@ namespace kagome::telemetry { boost::beast::lowest_layer_type & TelemetryConnectionImpl::stream_lowest_layer() { return secure_ ? boost::beast::get_lowest_layer( - *boost::relaxed_get(ws_)) + *boost::relaxed_get(ws_)) : boost::beast::get_lowest_layer( - *boost::relaxed_get(ws_)); + *boost::relaxed_get(ws_)); } template diff --git a/core/telemetry/impl/service_impl.cpp b/core/telemetry/impl/service_impl.cpp index 6576cf629a..9548dec18c 100644 --- a/core/telemetry/impl/service_impl.cpp +++ b/core/telemetry/impl/service_impl.cpp @@ -6,6 +6,7 @@ #include "telemetry/impl/service_impl.hpp" +#include #include #include @@ -30,6 +31,10 @@ namespace rapidjson { #include "telemetry/impl/telemetry_thread_pool.hpp" #include "utils/pool_handler_ready_make.hpp" +#ifdef __APPLE__ +#include +#endif + namespace { std::string json2string(rapidjson::Document &document) { rapidjson::StringBuffer buffer; @@ -37,6 +42,156 @@ namespace { document.Accept(writer); return buffer.GetString(); } + struct SysInfo { + std::optional cpu; + std::optional is_virtual_machine; + std::optional core_count; + std::optional memory; + std::optional linux_distro; + }; + +#ifdef __APPLE__ + SysInfo gatherMacSysInfo() { + SysInfo sysinfo; + + // Get the number of CPU cores + int mib[2]; + mib[0] = CTL_HW; + mib[1] = HW_NCPU; + int core_count; + size_t len = sizeof(core_count); + if (sysctl(mib, 2, &core_count, &len, nullptr, 0) == 0) { + sysinfo.core_count = core_count; + } + + // Get the CPU brand string + char cpu_brand[256]; + len = sizeof(cpu_brand); + mib[0] = CTL_HW; + mib[1] = HW_MACHINE; + if (sysctlbyname("machdep.cpu.brand_string", &cpu_brand, &len, nullptr, 0) + == 0) { + sysinfo.cpu = std::string(cpu_brand); + } + + // Get the memory size + uint64_t memory; + len = sizeof(memory); + mib[1] = HW_MEMSIZE; + if (sysctl(mib, 2, &memory, &len, nullptr, 0) == 0) { + sysinfo.memory = memory; + } + + // Get the macOS version name + std::array buffer; + std::string result; + std::unique_ptr pipe( + popen("sw_vers -productVersion", "r"), pclose); + if (pipe) { + while (fgets(buffer.data(), buffer.size(), pipe.get()) != nullptr) { + result += buffer.data(); + } + sysinfo.linux_distro = "MacOS " + result; + } + + // Check if running on a virtual machine + char hw_target[256]; + len = sizeof(hw_target); + mib[1] = HW_TARGET; + if (sysctl(mib, 2, &hw_target, &len, nullptr, 0) == 0) { + sysinfo.is_virtual_machine = + std::string(hw_target).find("VMware") != std::string::npos; + } + + return sysinfo; + } +#else + // Function to read file content into a string + std::optional read_file(const std::string &filepath) { + std::ifstream file(filepath); + if (not file.is_open()) { + return std::nullopt; + } + return {std::string((std::istreambuf_iterator(file)), + std::istreambuf_iterator())}; + } + + // Function to extract a single match using regex + template + std::optional extract(const std::string &data, const std::regex &pattern) { + std::smatch match; + if (std::regex_search(data, match, pattern)) { + if constexpr (std::is_same_v) { + return match[1].str(); + } else if constexpr (std::is_integral_v) { + return static_cast(std::stoull(match[1].str())); + } + } + return std::nullopt; + } + + SysInfo gatherLinuxSysInfo() { + const std::regex LINUX_REGEX_CPU(R"(^model name\s*:\s*([^\n]+))", + std::regex_constants::multiline); + const std::regex LINUX_REGEX_PHYSICAL_ID(R"(^physical id\s*:\s*(\d+))", + std::regex_constants::multiline); + const std::regex LINUX_REGEX_CORE_ID(R"(^core id\s*:\s*(\d+))", + std::regex_constants::multiline); + const std::regex LINUX_REGEX_HYPERVISOR(R"(^flags\s*:.+?\bhypervisor\b)", + std::regex_constants::multiline); + const std::regex LINUX_REGEX_MEMORY(R"(^MemTotal:\s*(\d+) kB)", + std::regex_constants::multiline); + const std::regex LINUX_REGEX_DISTRO(R"(PRETTY_NAME\s*=\s*\"([^\"]+)\")", + std::regex_constants::multiline); + + SysInfo sysinfo; + + if (auto cpuinfo = read_file("/proc/cpuinfo")) { + sysinfo.cpu = extract(*cpuinfo, LINUX_REGEX_CPU); + sysinfo.is_virtual_machine = + std::regex_search(*cpuinfo, LINUX_REGEX_HYPERVISOR); + + // Extract unique {physical_id, core_id} pairs + std::set > cores; + std::regex section_split("\n\n"); + std::sregex_token_iterator sections( + cpuinfo->begin(), cpuinfo->end(), section_split, -1); + std::sregex_token_iterator end; + for (; sections != end; ++sections) { + std::optional pid = + extract(*sections, LINUX_REGEX_PHYSICAL_ID); + std::optional cid = + extract(*sections, LINUX_REGEX_CORE_ID); + if (pid && cid) { + cores.emplace(*pid, *cid); + } + } + if (not cores.empty()) { + sysinfo.core_count = static_cast(cores.size()); + } + } + + if (auto meminfo = read_file("/proc/meminfo")) { + sysinfo.memory = + extract(*meminfo, LINUX_REGEX_MEMORY).value_or(0) * 1024; + } + + if (auto os_release = read_file("/etc/os-release")) { + sysinfo.linux_distro = + extract(*os_release, LINUX_REGEX_DISTRO); + } + + return sysinfo; + } +#endif + + SysInfo gatherSysInfo() { +#ifdef __APPLE__ + return gatherMacSysInfo(); +#else + return gatherLinuxSysInfo(); +#endif + } } // namespace namespace kagome::telemetry { @@ -258,6 +413,42 @@ namespace kagome::telemetry { .count()); rapidjson::Value payload(rapidjson::kObjectType); + rapidjson::Value sys_info_json(rapidjson::kObjectType); +#ifdef __APPLE__ + payload.AddMember("target_os", str_val("macos"), allocator); +#endif + struct utsname utsname_info = {}; + if (uname(&utsname_info) == 0) { +#ifndef __APPLE__ + payload.AddMember("target_os", str_val(utsname_info.sysname), allocator); +#endif + payload.AddMember( + "target_arch", str_val(utsname_info.machine), allocator); + sys_info_json.AddMember( + "linux_kernel", str_val(utsname_info.release), allocator); + } + const auto sys_info = gatherSysInfo(); + if (const auto &memory = sys_info.memory) { + sys_info_json.AddMember( + "memory", rapidjson::Value{}.SetUint64(*memory), allocator); + } + if (const auto &cpu = sys_info.cpu) { + sys_info_json.AddMember("cpu", str_val(*cpu), allocator); + } + if (const auto &core_count = sys_info.core_count) { + sys_info_json.AddMember( + "core_count", rapidjson::Value{}.SetUint(*core_count), allocator); + } + if (const auto &linux_distro = sys_info.linux_distro) { + sys_info_json.AddMember( + "linux_distro", str_val(*linux_distro), allocator); + } + if (const auto &is_virtual = sys_info.is_virtual_machine) { + sys_info_json.AddMember("is_virtual_machine", + rapidjson::Value{}.SetBool(*is_virtual), + allocator); + } + payload .AddMember( "authority", app_configuration_.roles().isAuthority(), allocator) @@ -270,7 +461,8 @@ namespace kagome::telemetry { .AddMember("network_id", str_val(host_.getId().toBase58()), allocator) .AddMember("startup_time", str_val(startup_time), allocator) .AddMember( - "version", str_val(app_configuration_.nodeVersion()), allocator); + "version", str_val(app_configuration_.nodeVersion()), allocator) + .AddMember("sysinfo", sys_info_json, allocator); greeting_json_.AddMember("id", 1, allocator) .AddMember("payload", payload, allocator) From f7a61ef4cd2b84ae3b05d86dd8c9c37240fcdd6a Mon Sep 17 00:00:00 2001 From: Ruslan Tushov Date: Fri, 29 Nov 2024 16:34:52 +0500 Subject: [PATCH 03/17] parallel apply blocks (#2284) Signed-off-by: turuslan Co-authored-by: kamilsa --- core/network/impl/synchronizer_impl.cpp | 143 +++++++++--------------- core/network/impl/synchronizer_impl.hpp | 5 +- 2 files changed, 56 insertions(+), 92 deletions(-) diff --git a/core/network/impl/synchronizer_impl.cpp b/core/network/impl/synchronizer_impl.cpp index eca774951e..a2f9292b37 100644 --- a/core/network/impl/synchronizer_impl.cpp +++ b/core/network/impl/synchronizer_impl.cpp @@ -30,6 +30,7 @@ #include "storage/trie/trie_storage.hpp" #include "storage/trie_pruner/trie_pruner.hpp" #include "utils/pool_handler_ready_make.hpp" +#include "utils/weak_macro.hpp" OUTCOME_CPP_DEFINE_CATEGORY(kagome::network, SynchronizerImpl::Error, e) { using E = kagome::network::SynchronizerImpl::Error; @@ -767,7 +768,7 @@ namespace kagome::network { BlockInfo(header.number, block.hash), peer_id); - self->generations_.emplace(header.number, block.hash); + self->generations_.emplace(header.blockInfo()); self->ancestry_.emplace(header.parent_hash, block.hash); some_blocks_added = true; @@ -899,53 +900,38 @@ namespace kagome::network { hash); processBlockAdditionResult(block_addition_result, hash, std::move(handler)); - postApplyBlock(hash); + ancestry_.erase(hash); + postApplyBlock(); } void SynchronizerImpl::applyNextBlock() { + auto any_block_applied = false; if (generations_.empty()) { - SL_TRACE(log_, "No block for applying"); - return; - } - - bool false_val = false; - if (not applying_in_progress_.compare_exchange_strong(false_val, true)) { - SL_TRACE(log_, "Applying in progress"); return; } - SL_TRACE(log_, "Begin applying"); - ::libp2p::common::MovableFinalAction cleanup([weak{weak_from_this()}] { - if (auto self = weak.lock()) { - SL_TRACE(self->log_, "End applying"); - self->applying_in_progress_ = false; - } - }); - - primitives::BlockHash hash; - - while (true) { - auto generation_node = generations_.extract(generations_.begin()); - if (generation_node) { - hash = generation_node.mapped(); - break; - } - if (generations_.empty()) { - SL_TRACE(log_, "No block for applying"); - return; + while (not generations_.empty()) { + auto block_info = *generations_.begin(); + auto pop = [&] { generations_.erase(generations_.begin()); }; + auto it = known_blocks_.find(block_info.hash); + if (it == known_blocks_.end()) { + pop(); + continue; } - } - - if (auto it = known_blocks_.find(hash); it != known_blocks_.end()) { auto &block_data = it->second.data; BOOST_ASSERT(block_data.header.has_value()); - const BlockInfo block_info(block_data.header->number, block_data.hash); + auto parent = block_data.header->parentInfo(); + if (parent and not block_tree_->has(parent->hash)) { + break; + } + pop(); + any_block_applied = true; const auto &last_finalized_block = block_tree_->getLastFinalized(); SyncResultHandler handler; if (watched_blocks_number_ == block_data.header->number) { - if (auto wbn_node = watched_blocks_.extract(hash)) { + if (auto wbn_node = watched_blocks_.extract(block_info.hash)) { handler = std::move(wbn_node.mapped()); } } @@ -953,7 +939,7 @@ namespace kagome::network { // Skip applied and finalized blocks and // discard side-chain below last finalized if (block_data.header->number <= last_finalized_block.number) { - if (not block_tree_->has(hash)) { + if (not block_tree_->has(block_info.hash)) { auto n = discardBlock(block_data.hash); SL_WARN( log_, @@ -966,18 +952,13 @@ namespace kagome::network { } } else { - auto callback = - [wself{weak_from_this()}, - hash, - handler{std::move(handler)}, - cleanup = std::make_shared(std::move(cleanup))]( - auto &&block_addition_result) mutable { - cleanup.reset(); - if (auto self = wself.lock()) { - self->post_block_addition( - std::move(block_addition_result), std::move(handler), hash); - } - }; + auto callback = [WEAK_SELF, block_info, handler{std::move(handler)}]( + auto &&block_addition_result) mutable { + WEAK_LOCK(self); + self->post_block_addition(std::move(block_addition_result), + std::move(handler), + block_info.hash); + }; if (sync_method_ == application::SyncMethod::Full) { // Regular syncing @@ -1013,8 +994,11 @@ namespace kagome::network { } return; } + ancestry_.erase(block_info.hash); + } + if (any_block_applied) { + postApplyBlock(); } - postApplyBlock(hash); } void SynchronizerImpl::processBlockAdditionResult( @@ -1062,9 +1046,7 @@ namespace kagome::network { } } - void SynchronizerImpl::postApplyBlock(const primitives::BlockHash &hash) { - ancestry_.erase(hash); - + void SynchronizerImpl::postApplyBlock() { auto minPreloadedBlockAmount = sync_method_ == application::SyncMethod::Full ? kMinPreloadedBlockAmount : kMinPreloadedBlockAmountForFastSyncing; @@ -1118,29 +1100,20 @@ namespace kagome::network { void SynchronizerImpl::prune(const primitives::BlockInfo &finalized_block) { // Remove blocks whose numbers less finalized one while (not generations_.empty()) { - auto generation_node = generations_.extract(generations_.begin()); - if (generation_node) { - const auto &number = generation_node.key(); - if (number >= finalized_block.number) { - break; - } - const auto &hash = generation_node.mapped(); - notifySubscribers({number, hash}, Error::DISCARDED_BLOCK); - - known_blocks_.erase(hash); - ancestry_.erase(hash); + auto block_info = *generations_.begin(); + if (block_info.number > finalized_block.number) { + break; } - } - - // Remove blocks whose numbers equal finalized one, excluding finalized - // one - auto range = generations_.equal_range(finalized_block.number); - for (auto it = range.first; it != range.second;) { - auto cit = it++; - const auto &hash = cit->second; - if (hash != finalized_block.hash) { - discardBlock(hash); + if (block_info.number == finalized_block.number) { + if (block_info.hash != finalized_block.hash) { + discardBlock(block_info.hash); + } + continue; } + generations_.erase(*generations_.begin()); + notifySubscribers(block_info, Error::DISCARDED_BLOCK); + known_blocks_.erase(block_info.hash); + ancestry_.erase(block_info.hash); } metric_import_queue_length_->set(known_blocks_.size()); @@ -1169,18 +1142,13 @@ namespace kagome::network { for (auto g_it = generations_.rbegin(); g_it != generations_.rend(); ++g_it) { - const auto &hash = g_it->second; - - auto b_it = known_blocks_.find(hash); + auto &block_info = *g_it; + auto b_it = known_blocks_.find(block_info.hash); if (b_it == known_blocks_.end()) { - SL_TRACE(log_, - "Block {} is unknown. Go to next one", - primitives::BlockInfo(g_it->first, hash)); + SL_TRACE(log_, "Block {} is unknown. Go to next one", block_info); continue; } - primitives::BlockInfo block_info(g_it->first, hash); - auto &peers = b_it->second.peers; if (peers.empty()) { SL_TRACE( @@ -1194,10 +1162,7 @@ namespace kagome::network { auto &peer_id = *cp_it; if (busy_peers_.find(peer_id) != busy_peers_.end()) { - SL_TRACE(log_, - "Peer {} for block {} is busy", - peer_id, - primitives::BlockInfo(g_it->first, hash)); + SL_TRACE(log_, "Peer {} for block {} is busy", peer_id, block_info); continue; } @@ -1229,16 +1194,16 @@ namespace kagome::network { }; if (sync_method_ == application::SyncMethod::Full) { - auto lower = generations_.begin()->first; - auto upper = generations_.rbegin()->first + 1; - auto hint = generations_.rbegin()->first; + auto lower = generations_.begin()->number; + auto upper = generations_.rbegin()->number + 1; + auto hint = generations_.rbegin()->number; SL_DEBUG( log_, "Start to find common block with {} in #{}..#{} to fill queue", peer_id, - generations_.begin()->first, - generations_.rbegin()->first); + generations_.begin()->number, + generations_.rbegin()->number); findCommonBlock( peer_id, lower, @@ -1278,7 +1243,7 @@ namespace kagome::network { SL_TRACE(log_, "Block {} doesn't have appropriate peer. Go to next one", - primitives::BlockInfo(g_it->first, hash)); + block_info); } SL_TRACE(log_, "End asking portion of blocks: none"); diff --git a/core/network/impl/synchronizer_impl.hpp b/core/network/impl/synchronizer_impl.hpp index c139387ecc..6f932976c4 100644 --- a/core/network/impl/synchronizer_impl.hpp +++ b/core/network/impl/synchronizer_impl.hpp @@ -191,7 +191,7 @@ namespace kagome::network { SyncResultHandler &&handler); private: - void postApplyBlock(const primitives::BlockHash &hash); + void postApplyBlock(); void processBlockAdditionResult(outcome::result block_addition_result, const primitives::BlockHash &hash, SyncResultHandler &&handler); @@ -292,7 +292,7 @@ namespace kagome::network { std::unordered_map known_blocks_; // Blocks grouped by number - std::multimap generations_; + std::set generations_; // Links parent->child std::unordered_multimap @@ -307,7 +307,6 @@ namespace kagome::network { std::multimap subscriptions_; - std::atomic_bool applying_in_progress_ = false; std::atomic_bool asking_blocks_portion_in_progress_ = false; std::set busy_peers_; std::unordered_set load_blocks_; From ced71b32858033f43eef22fc7e70dc68039d71ad Mon Sep 17 00:00:00 2001 From: Boris <31869190+ErakhtinB@users.noreply.github.com> Date: Fri, 29 Nov 2024 17:43:58 +0500 Subject: [PATCH 04/17] candidates_pending_availability added (#2253) --- .../runtime_api/impl/parachain_host.cpp | 13 +++++++++++ .../runtime_api/impl/parachain_host.hpp | 7 ++++++ core/runtime/runtime_api/parachain_host.hpp | 10 +++++++++ test/core/runtime/binaryen/CMakeLists.txt | 1 + test/core/runtime/binaryen/parachain_test.cpp | 22 +++++++++++++++++++ .../mock/core/runtime/parachain_host_mock.hpp | 5 +++++ 6 files changed, 58 insertions(+) diff --git a/core/runtime/runtime_api/impl/parachain_host.cpp b/core/runtime/runtime_api/impl/parachain_host.cpp index a69fb90771..9a264869e9 100644 --- a/core/runtime/runtime_api/impl/parachain_host.cpp +++ b/core/runtime/runtime_api/impl/parachain_host.cpp @@ -150,6 +150,18 @@ namespace kagome::runtime { return *ref; } + outcome::result>> + ParachainHostImpl::candidates_pending_availability( + const primitives::BlockHash &block, ParachainId id) { + OUTCOME_TRY(ref, + candidates_pending_availability_.call( + *executor_, + block, + "ParachainHost_candidates_pending_availability", + id)); + return std::move(*ref); + } + outcome::result> ParachainHostImpl::candidate_events(const primitives::BlockHash &block) { OUTCOME_TRY(ref, @@ -214,6 +226,7 @@ namespace kagome::runtime { availability_cores_.erase(blocks); session_index_for_child_.erase(blocks); candidate_pending_availability_.erase(blocks); + candidates_pending_availability_.erase(blocks); candidate_events_.erase(blocks); session_info_.erase(blocks); dmq_contents_.erase(blocks); diff --git a/core/runtime/runtime_api/impl/parachain_host.hpp b/core/runtime/runtime_api/impl/parachain_host.hpp index a00b904e7b..103e58e4a1 100644 --- a/core/runtime/runtime_api/impl/parachain_host.hpp +++ b/core/runtime/runtime_api/impl/parachain_host.hpp @@ -68,6 +68,10 @@ namespace kagome::runtime { candidate_pending_availability(const primitives::BlockHash &block, ParachainId id) override; + outcome::result>> + candidates_pending_availability(const primitives::BlockHash &block, + ParachainId id) override; + outcome::result> candidate_events( const primitives::BlockHash &block) override; @@ -144,6 +148,9 @@ namespace kagome::runtime { }; RuntimeApiLruBlockArg> candidate_pending_availability_{10}; + RuntimeApiLruBlockArg>> + candidates_pending_availability_{10}; RuntimeApiLruBlock> candidate_events_{10}; RuntimeApiLruBlockArg> session_info_{10}; diff --git a/core/runtime/runtime_api/parachain_host.hpp b/core/runtime/runtime_api/parachain_host.hpp index c008d10bbd..bf852b8eae 100644 --- a/core/runtime/runtime_api/parachain_host.hpp +++ b/core/runtime/runtime_api/parachain_host.hpp @@ -140,6 +140,16 @@ namespace kagome::runtime { candidate_pending_availability(const primitives::BlockHash &block, ParachainId id) = 0; + /** + * @brief Get the receipts of all candidates pending availability. + * @param id parachain id + * @return vector of CommittedCandidateReceipt + */ + virtual outcome::result< + std::vector>> + candidates_pending_availability(const primitives::BlockHash &block, + ParachainId id) = 0; + /// /** * @brief Get a vector of events concerning candidates that occurred within diff --git a/test/core/runtime/binaryen/CMakeLists.txt b/test/core/runtime/binaryen/CMakeLists.txt index 22dea62cd1..6f65ecf3bb 100644 --- a/test/core/runtime/binaryen/CMakeLists.txt +++ b/test/core/runtime/binaryen/CMakeLists.txt @@ -64,6 +64,7 @@ target_link_libraries(binaryen_parachain_test hasher pbkdf2_provider network + logger_for_tests ) addtest(metadata_test diff --git a/test/core/runtime/binaryen/parachain_test.cpp b/test/core/runtime/binaryen/parachain_test.cpp index 7bf5b8af81..ae5e04384c 100644 --- a/test/core/runtime/binaryen/parachain_test.cpp +++ b/test/core/runtime/binaryen/parachain_test.cpp @@ -12,6 +12,7 @@ #include "host_api/impl/host_api_impl.hpp" #include "runtime/binaryen/memory_impl.hpp" #include "testutil/outcome.hpp" +#include "testutil/prepare_loggers.hpp" using kagome::common::Buffer; using kagome::host_api::HostApiImpl; @@ -30,8 +31,19 @@ using ::testing::Return; namespace fs = kagome::filesystem; +/** + * @class ParachainHostTest + * + * All tests are currently disabled as they require storage + * with information for runtime. + * + */ class ParachainHostTest : public BinaryenRuntimeTest { public: + static void SetUpTestCase() { + testutil::prepareLoggers(); + } + void SetUp() override { BinaryenRuntimeTest::SetUp(); @@ -167,6 +179,16 @@ TEST_F(ParachainHostTest, DISABLED_CandidatePendingAvailabilityTest) { ASSERT_TRUE(api_->candidate_pending_availability("block_hash"_hash256, id)); } +/** + * @given initialized parachain host api + * @when candidates_pending_availability() is invoked + * @then successful result is returned + */ +TEST_F(ParachainHostTest, DISABLED_CandidatesPendingAvailabilityTest) { + auto id = createParachainId(); + ASSERT_TRUE(api_->candidates_pending_availability("block_hash"_hash256, id)); +} + /** * @given initialized parachain host api * @when candidate_events() is invoked diff --git a/test/mock/core/runtime/parachain_host_mock.hpp b/test/mock/core/runtime/parachain_host_mock.hpp index 9bc83525cc..c3809ab968 100644 --- a/test/mock/core/runtime/parachain_host_mock.hpp +++ b/test/mock/core/runtime/parachain_host_mock.hpp @@ -80,6 +80,11 @@ namespace kagome::runtime { (const primitives::BlockHash &, ParachainId), (override)); + MOCK_METHOD(outcome::result>>, + candidates_pending_availability, + (const primitives::BlockHash &, ParachainId), + (override)); + MOCK_METHOD(outcome::result>, candidate_events, (const primitives::BlockHash &), From a451b24080dca040c4e365db197b0a450fcf9d9c Mon Sep 17 00:00:00 2001 From: kamilsa Date: Fri, 29 Nov 2024 21:35:22 +0500 Subject: [PATCH 05/17] Small improvements (#2296) * Make compiled as default option * Reduce some logs * Update hunter --------- Co-authored-by: Kirill Azovtsev --- .github/workflows/test.yml | 117 ++++++++++++------ cmake/Hunter/config.cmake | 25 +--- cmake/Hunter/hunter-gate-url.cmake | 4 +- cmake/toolchain/clang-16_cxx20.cmake | 2 - cmake/toolchain/clang-19_cxx20.cmake | 2 + ..._cxx20.cmake => clang-19_mold_cxx20.cmake} | 2 +- .../{clang-16.cmake => clang-19.cmake} | 20 +-- .../impl/app_configuration_impl.cpp | 2 +- core/crypto/twox/twox.cpp | 6 +- .../approval/approval_distribution.cpp | 61 ++++----- .../availability/fetch/fetch_impl.cpp | 2 +- .../validator/impl/parachain_processor.cpp | 2 +- .../statement_distribution.cpp | 2 +- housekeeping/docker/kagome-dev/Makefile | 30 +++-- .../kagome-dev/kagome_builder.Dockerfile | 1 + housekeeping/macos/dependency.sh | 2 +- scripts/.env | 2 +- scripts/init.sh | 2 +- 18 files changed, 158 insertions(+), 126 deletions(-) delete mode 100644 cmake/toolchain/clang-16_cxx20.cmake create mode 100644 cmake/toolchain/clang-19_cxx20.cmake rename cmake/toolchain/{clang-16_mold_cxx20.cmake => clang-19_mold_cxx20.cmake} (62%) rename cmake/toolchain/compiler/{clang-16.cmake => clang-19.cmake} (75%) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 02a064ec9d..76aa425633 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -69,15 +69,20 @@ env: jobs: MacOS: runs-on: macos-15 - timeout-minutes: 120 + timeout-minutes: 240 strategy: fail-fast: false matrix: options: - name: "MacOS: Build Debug" build_type: "Debug" + wasm_compiler: "WasmEdge" - name: "MacOS: Build Release" build_type: "Release" + wasm_compiler: "WasmEdge" + - name: "MacOS WAVM: Build Debug" + build_type: "Debug" + wasm_compiler: "WAVM" name: "${{ matrix.options.name }}" steps: - uses: actions/checkout@v4 @@ -91,12 +96,11 @@ jobs: env: KAGOME_MAC_CI: 1 - name: build - run: ./housekeeping/make_build.sh -G Ninja -DCMAKE_BUILD_TYPE=${{ matrix.options.build_type }} -DCLEAR_OBJS=ON -DCOVERAGE=OFF -DWASM_COMPILER=WasmEdge -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/cxx20.cmake + run: ./housekeeping/make_build.sh -G Ninja -DCMAKE_BUILD_TYPE=${{ matrix.options.build_type }} -DCLEAR_OBJS=ON -DCOVERAGE=OFF -DWASM_COMPILER=${{ matrix.options.wasm_compiler }} -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/cxx20.cmake env: KAGOME_IN_DOCKER: 0 KAGOME_MAC_CI: 1 - Linux: if: false # Need to fix # ${{ @@ -106,12 +110,12 @@ jobs: fail-fast: false matrix: options: - - name: "Linux: clang-16 External Project" - run: ./housekeeping/make_external_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=../../cmake/toolchain/clang-16_cxx20.cmake + - name: "Linux: clang-19 External Project" + run: ./housekeeping/make_external_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=../../cmake/toolchain/clang-19_cxx20.cmake name: "${{ matrix.options.name }}" runs-on: ubuntu-24.04 timeout-minutes: 120 - container: qdrvm/kagome-dev:9-minideb + container: qdrvm/kagome_builder:99c3519_rust1.81.0_gcc13_llvm19 steps: - uses: actions/checkout@v4 - uses: actions/cache@v4 @@ -133,44 +137,12 @@ jobs: df -m || true du -hd1 /__w /github || true - Linux-self-hosted: - strategy: - fail-fast: false - matrix: - options: - - name: "Self-hosted: Linux: gcc-13 ASAN" - run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-13_cxx20.cmake -DASAN=ON - - name: "Self-hosted: Linux: clang-16 TSAN WAVM" - run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-16_cxx20.cmake -DTSAN=ON -DWASM_COMPILER=WAVM - - name: "Self-hosted: Linux: clang-16 UBSAN" - run: ./housekeeping/make_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-16_cxx20.cmake -DUBSAN=ON -DUBSAN_TRAP=OFF -DUBSAN_ABORT=ON - env: UBSAN_OPTIONS=print_stacktrace=1 - # Need to fix - # - name: "Self-hosted: Linux: clang-16 External Project" - # run: ./housekeeping/make_external_build.sh -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=../../cmake/toolchain/clang-16_cxx20.cmake - - name: "${{ matrix.options.name }}" - runs-on: [ actions-runner-controller ] - timeout-minutes: 120 - container: qdrvm/kagome-dev:9-minideb - steps: - - uses: actions/checkout@v4 - - uses: actions/cache@v4 - if: ${{ env.USE_CACHE == 'true' }} - with: - path: ${{ env.CACHE_PATHS }} - key: ${{ github.job }}-${{ matrix.options.name }}-${{ env.CACHE_VERSION }} - - name: Install mold - run: ./housekeeping/ci_install_mold.sh --make-default - - name: "${{ matrix.options.name }}" - run: "${{ matrix.options.run }}" - coverage-self-hosted: if: false # ${{ github.ref == 'refs/heads/master' || startsWith( github.ref, 'refs/tags/') || contains( github.event.pull_request.labels.*.name, 'Non-master self-hosted') }} name: "Self-hosted: Linux: gcc-13 coverage/sonar" runs-on: [ actions-runner-controller ] timeout-minutes: 120 - container: qdrvm/kagome-dev:9-minideb + container: qdrvm/kagome_builder:99c3519_rust1.81.0_gcc13_llvm19 steps: - uses: actions/checkout@v4 - uses: actions/cache@v4 @@ -202,6 +174,73 @@ jobs: BRANCH_NAME: ${{ github.ref }} run: if [ "$SONAR_TOKEN" != "null" -a "$GITHUB_USERNAME" != "null" -a "$GITHUB_TOKEN" != "null" ]; then ./housekeeping/sonar.sh; else echo "Some secret undefined. Step passed..."; fi + kagome_dev_docker_build_sanitizers: + strategy: + fail-fast: false + matrix: + options: + - name: "Self-hosted: Linux: gcc-13 ASAN" + params: -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-13_cxx20.cmake -DASAN=ON + - name: "Self-hosted: Linux: clang-19 TSAN WAVM" + params: -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-19_cxx20.cmake -DTSAN=ON + - name: "Self-hosted: Linux: clang-19 UBSAN" + params: -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/clang-19_cxx20.cmake -DUBSAN=ON -DUBSAN_TRAP=OFF -DUBSAN_ABORT=ON + + name: "${{ matrix.options.name }}" + runs-on: [ actions-runner-controller ] + timeout-minutes: 180 + steps: + - name: "Checkout repository" + uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: "Get master branch" + if: github.ref != 'refs/heads/master' + run: git fetch origin master:master || true + + - name: "Get commit version" + working-directory: ./housekeeping/docker/kagome-dev + run: make get_versions + + - name: "Check version" + working-directory: ./housekeeping/docker/kagome-dev + run: | + SHORT_COMMIT_HASH=$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2) + echo "short_commit_hash=${SHORT_COMMIT_HASH}" | tee $GITHUB_ENV + + - name: "Cache dependencies" + id: cache-restore + if: ${{ env.USE_CACHE == 'true' }} + uses: actions/cache/restore@v4 + with: + path: ${{ env.CACHE_PATH }} + key: ${{ github.job }}-${{ env.CACHE_VERSION }}-sanitizers-${{ env.short_commit_hash }} + restore-keys: | + ${{ github.job }}-${{ env.CACHE_VERSION }}-sanitizers- + + - name: "Build target" + working-directory: ./housekeeping/docker/kagome-dev + run: + make kagome_dev_docker_build_sanitizers \ + GITHUB_HUNTER_USERNAME=${{ secrets.HUNTER_USERNAME }} \ + GITHUB_HUNTER_TOKEN=${{ secrets.HUNTER_TOKEN }} \ + BUILDER_IMAGE_TAG=${{ env.BUILDER_LATEST_TAG }} \ + CI="true" + SAN_PARAMS="${{ matrix.options.params }}" + + - name: "Cleaning cache" + run: | + find ${{ env.CACHE_PATH }} -name '*.pdf' -exec rm {} \; + + - name: "Always Save Cache" + id: cache-save + if: always() && (steps.cache-restore.outputs.cache-hit != 'true' || env.package_exist != 'True') + uses: actions/cache/save@v4 + with: + path: ${{ env.CACHE_PATH }} + key: ${{ steps.cache-restore.outputs.cache-primary-key }} + kagome_dev_docker_build_tidy: runs-on: ubuntu-24.04 timeout-minutes: 600 # TODO(xDimon): revert after merge PR#2208 diff --git a/cmake/Hunter/config.cmake b/cmake/Hunter/config.cmake index 8c4c514445..bbd1efe960 100644 --- a/cmake/Hunter/config.cmake +++ b/cmake/Hunter/config.cmake @@ -25,13 +25,6 @@ hunter_config( KEEP_PACKAGE_SOURCES ) -hunter_config( - benchmark - URL https://github.com/google/benchmark/archive/refs/tags/v1.8.3.zip - SHA1 bf9870756ee3f8d2d3b346b24ee3600a41c74d3d - CMAKE_ARGS BENCHMARK_ENABLE_TESTING=OFF -) - hunter_config( rocksdb VERSION 9.6.1 @@ -99,8 +92,8 @@ endif () hunter_config( kagome-crates - URL https://github.com/qdrvm/kagome-crates/archive/refs/tags/1.0.2.tar.gz - SHA1 946c48508545380e155ab831be54228b916544d3 + URL https://github.com/qdrvm/kagome-crates/archive/refs/tags/v1.0.3.tar.gz + SHA1 4207446a0e45764b814805821aa6860924b03cb7 ) hunter_config( @@ -112,29 +105,23 @@ hunter_config( hunter_config( libp2p - URL https://github.com/libp2p/cpp-libp2p/archive/c96d45f792fafd6970a7e37ec816b02a9167e2b6.zip - SHA1 884932112bc75996eeecd4a7bbcb932565fe9859 + VERSION 0.1.28 ) hunter_config( scale - URL https://github.com/qdrvm/scale-codec-cpp/archive/e1a3c7afafc2eeda0c8e2daed08da6b7789f44b3.zip - SHA1 b56bcda34fb0d293c88d8b642b2f3fdc2d16a3e5 + VERSION 1.1.4 ) hunter_config( erasure_coding_crust -# VERSION 0.0.8 - URL https://github.com/qdrvm/erasure-coding-crust/archive/refs/tags/v0.0.8.tar.gz - SHA1 6bcdb6327f5da2dcec5c70f2fa63b95a44925af0 + VERSION 0.0.8 KEEP_PACKAGE_SOURCES ) hunter_config( soralog -# VERSION 0.2.4 - URL https://github.com/qdrvm/soralog/archive/refs/tags/v0.2.4.tar.gz - SHA1 1de495d8a3a73c1e940be3fdddf263a2d673aec1 + VERSION 0.2.4 KEEP_PACKAGE_SOURCES ) diff --git a/cmake/Hunter/hunter-gate-url.cmake b/cmake/Hunter/hunter-gate-url.cmake index 1be9836f94..83591dcef4 100644 --- a/cmake/Hunter/hunter-gate-url.cmake +++ b/cmake/Hunter/hunter-gate-url.cmake @@ -1,5 +1,5 @@ HunterGate( - URL https://github.com/qdrvm/hunter/archive/refs/tags/v0.25.3-qdrvm23.zip - SHA1 3fb58496e599fa3d35e94d1810324c6b379029f1 + URL https://github.com/qdrvm/hunter/archive/refs/tags/v0.25.3-qdrvm25.zip + SHA1 59c66ff04ebd2cbdf86c3b996d38d4be6eaaa78b LOCAL ) \ No newline at end of file diff --git a/cmake/toolchain/clang-16_cxx20.cmake b/cmake/toolchain/clang-16_cxx20.cmake deleted file mode 100644 index 16081adfc7..0000000000 --- a/cmake/toolchain/clang-16_cxx20.cmake +++ /dev/null @@ -1,2 +0,0 @@ -include(${CMAKE_CURRENT_LIST_DIR}/compiler/clang-16.cmake) -include(${CMAKE_CURRENT_LIST_DIR}/cxx20.cmake) diff --git a/cmake/toolchain/clang-19_cxx20.cmake b/cmake/toolchain/clang-19_cxx20.cmake new file mode 100644 index 0000000000..ff37710c7c --- /dev/null +++ b/cmake/toolchain/clang-19_cxx20.cmake @@ -0,0 +1,2 @@ +include(${CMAKE_CURRENT_LIST_DIR}/compiler/clang-19.cmake) +include(${CMAKE_CURRENT_LIST_DIR}/cxx20.cmake) diff --git a/cmake/toolchain/clang-16_mold_cxx20.cmake b/cmake/toolchain/clang-19_mold_cxx20.cmake similarity index 62% rename from cmake/toolchain/clang-16_mold_cxx20.cmake rename to cmake/toolchain/clang-19_mold_cxx20.cmake index 86c85ef3b4..2bfc4cae5f 100644 --- a/cmake/toolchain/clang-16_mold_cxx20.cmake +++ b/cmake/toolchain/clang-19_mold_cxx20.cmake @@ -1,3 +1,3 @@ -include(${CMAKE_CURRENT_LIST_DIR}/compiler/clang-16.cmake) +include(${CMAKE_CURRENT_LIST_DIR}/compiler/clang-19.cmake) include(${CMAKE_CURRENT_LIST_DIR}/cxx20.cmake) include(${CMAKE_CURRENT_LIST_DIR}/linker/mold.cmake) diff --git a/cmake/toolchain/compiler/clang-16.cmake b/cmake/toolchain/compiler/clang-19.cmake similarity index 75% rename from cmake/toolchain/compiler/clang-16.cmake rename to cmake/toolchain/compiler/clang-19.cmake index 61c25e46f2..ce8062443f 100644 --- a/cmake/toolchain/compiler/clang-16.cmake +++ b/cmake/toolchain/compiler/clang-19.cmake @@ -1,7 +1,7 @@ -if(DEFINED POLLY_COMPILER_CLANG_16_CMAKE) +if(DEFINED POLLY_COMPILER_CLANG_19_CMAKE) return() else() - set(POLLY_COMPILER_CLANG_16_CMAKE 1) + set(POLLY_COMPILER_CLANG_19_CMAKE 1) endif() include(${CMAKE_CURRENT_LIST_DIR}/../../print.cmake) @@ -13,29 +13,29 @@ if(XCODE_VERSION) fatal_error(${_err}) endif() -find_program(CMAKE_C_COMPILER clang-16) -find_program(CMAKE_CXX_COMPILER clang++-16) +find_program(CMAKE_C_COMPILER clang-19) +find_program(CMAKE_CXX_COMPILER clang++-19) if (CMAKE_CXX_COMPILER STREQUAL "CMAKE_CXX_COMPILER-NOTFOUND") - message(STATUS "clang++-16 not found, checking clang++") + message(STATUS "clang++-19 not found, checking clang++") cmake_path(GET CMAKE_C_COMPILER PARENT_PATH compiler_path) message(STATUS "Assumed compiler path: ${compiler_path}") - # clang++-16 doesn't always exist + # clang++-19 doesn't always exist find_program(CMAKE_CXX_COMPILER clang++ PATHS "${compiler_path}" NO_DEFAULT_PATH REQUIRED) execute_process(COMMAND "${CMAKE_CXX_COMPILER}" --version OUTPUT_VARIABLE compiler_version_output) string(REGEX MATCH "clang version ([0-9]+)\\.[0-9]+\\.[0-9]+" compiler_version "${compiler_version_output}") - if (NOT CMAKE_MATCH_1 STREQUAL "16") - message(FATAL_ERROR "Found clang++ version ${CMAKE_MATCH_1}, 16 is required") + if (NOT CMAKE_MATCH_1 STREQUAL "19") + message(FATAL_ERROR "Found clang++ version ${CMAKE_MATCH_1}, 19 is required") endif() endif() if(NOT CMAKE_C_COMPILER) - fatal_error("clang-16 not found") + fatal_error("clang-19 not found") endif() if(NOT CMAKE_CXX_COMPILER) - fatal_error("clang++-16 not found") + fatal_error("clang++-19 not found") endif() set( diff --git a/core/application/impl/app_configuration_impl.cpp b/core/application/impl/app_configuration_impl.cpp index 9ca33b02e3..c19b473cc0 100644 --- a/core/application/impl/app_configuration_impl.cpp +++ b/core/application/impl/app_configuration_impl.cpp @@ -90,7 +90,7 @@ namespace { }(); const auto def_sync_method = kagome::application::SyncMethod::Full; const auto def_runtime_exec_method = - kagome::application::AppConfiguration::RuntimeExecutionMethod::Interpret; + kagome::application::AppConfiguration::RuntimeExecutionMethod::Compile; const auto def_runtime_interpreter = kagome::application::AppConfiguration::RuntimeInterpreter::WasmEdge; const auto def_purge_wavm_cache_ = false; diff --git a/core/crypto/twox/twox.cpp b/core/crypto/twox/twox.cpp index ac47c25e55..13903e2a7b 100644 --- a/core/crypto/twox/twox.cpp +++ b/core/crypto/twox/twox.cpp @@ -39,8 +39,11 @@ namespace kagome::crypto { } void make_twox256(const uint8_t *in, uint32_t len, uint8_t *out) { + // Ensure the buffer is aligned to the boundary required for uint64_t + // (required for happy UBSAN) + std::array aligned_out{}; // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) - auto *ptr = reinterpret_cast(out); + auto *ptr = reinterpret_cast(aligned_out.data()); // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) ptr[0] = XXH64(in, len, 0); // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) @@ -49,6 +52,7 @@ namespace kagome::crypto { ptr[2] = XXH64(in, len, 2); // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) ptr[3] = XXH64(in, len, 3); + std::memcpy(out, aligned_out.data(), 4 * sizeof(uint64_t)); } common::Hash256 make_twox256(common::BufferView buf) { diff --git a/core/parachain/approval/approval_distribution.cpp b/core/parachain/approval/approval_distribution.cpp index 03f701184b..13e3622bb1 100644 --- a/core/parachain/approval/approval_distribution.cpp +++ b/core/parachain/approval/approval_distribution.cpp @@ -588,7 +588,7 @@ namespace kagome::parachain { common::MainThreadPool &main_thread_pool, LazySPtr dispute_coordinator) : approval_thread_handler_{poolHandlerReadyMake( - this, app_state_manager, approval_thread_pool, logger_)}, + this, app_state_manager, approval_thread_pool, logger_)}, worker_pool_handler_{worker_thread_pool.handler(*app_state_manager)}, parachain_host_(std::move(parachain_host)), slots_util_(slots_util), @@ -1657,17 +1657,18 @@ namespace kagome::parachain { const auto &candidate_receipt = hashed_candidate.get(); if (!opt_result) { // Unavailable - self->logger_->warn( - "No available parachain data.(session index={}, candidate " - "hash={}, relay block hash={})", - session_index, - hashed_candidate.getHash(), - relay_block_hash); + SL_DEBUG(self->logger_, + "No available parachain data. (session index={}, " + "candidate hash={}, relay block hash={})", + session_index, + hashed_candidate.getHash(), + relay_block_hash); return; } if (opt_result->has_error()) { - self->logger_->warn( + SL_WARN( + self->logger_, "Parachain data recovery failed.(error={}, session index={}, " "candidate hash={}, relay block hash={})", opt_result->error(), @@ -1685,7 +1686,8 @@ namespace kagome::parachain { auto result = self->parachain_host_->validation_code_by_hash( block_hash, candidate_receipt.descriptor.validation_code_hash); if (result.has_error() || !result.value()) { - self->logger_->warn( + SL_WARN( + self->logger_, "Approval state is failed. Block hash {}, session index {}, " "validator index {}, relay parent {}", block_hash, @@ -1695,12 +1697,12 @@ namespace kagome::parachain { return; /// ApprovalState::failed } - self->logger_->info( - "Make exhaustive validation. Candidate hash {}, validator index " - "{}, block hash {}", - hashed_candidate.getHash(), - validator_index, - block_hash); + SL_DEBUG(self->logger_, + "Make exhaustive validation. Candidate hash {}, validator " + "index {}, block hash {}", + hashed_candidate.getHash(), + validator_index, + block_hash); runtime::ValidationCode &validation_code = *result.value(); @@ -1726,12 +1728,12 @@ namespace kagome::parachain { } }); if (outcome.has_error()) { - self->logger_->warn( - "Approval validation failed.(parachain id={}, relay " - "parent={}, error={})", - candidate_receipt.descriptor.para_id, - candidate_receipt.descriptor.relay_parent, - outcome.error()); + SL_WARN(self->logger_, + "Approval validation failed.(parachain id={}, relay " + "parent={}, error={})", + candidate_receipt.descriptor.para_id, + candidate_receipt.descriptor.relay_parent, + outcome.error()); self->dispute_coordinator_.get()->issueLocalStatement( session_index, hashed_candidate.getHash(), @@ -2670,10 +2672,10 @@ namespace kagome::parachain { REINVOKE( *main_pool_handler_, runDistributeApproval, vote, std::move(peers)); - SL_INFO(logger_, - "Sending an approval to peers. (block={}, num peers={})", - vote.payload.payload.block_hash, - peers.size()); + SL_DEBUG(logger_, + "Sending an approval to peers. (block={}, num peers={})", + vote.payload.payload.block_hash, + peers.size()); router_->getValidationProtocol()->write(peers, network::vstaging::Approvals{ @@ -2892,9 +2894,10 @@ namespace kagome::parachain { }; return approval::min_or_some( e.next_no_show, - (e.last_assignment_tick ? filter( - *e.last_assignment_tick + kApprovalDelay, tick_now) - : std::optional{})); + (e.last_assignment_tick + ? filter(*e.last_assignment_tick + kApprovalDelay, + tick_now) + : std::optional{})); }, [&](const approval::PendingRequiredTranche &e) { std::optional next_announced{}; @@ -3160,7 +3163,7 @@ namespace kagome::parachain { auto opt_candidate_entry = storedCandidateEntries().get(candidate_hash); if (!opt_block_entry || !opt_candidate_entry) { - SL_ERROR(logger_, "Block entry or candidate entry not exists."); + SL_TRACE(logger_, "Block entry or candidate entry not exists."); return; } diff --git a/core/parachain/availability/fetch/fetch_impl.cpp b/core/parachain/availability/fetch/fetch_impl.cpp index f70297233c..2ffdeb6246 100644 --- a/core/parachain/availability/fetch/fetch_impl.cpp +++ b/core/parachain/availability/fetch/fetch_impl.cpp @@ -154,7 +154,7 @@ namespace kagome::parachain { } return; } - SL_WARN(log(), + SL_DEBUG(log(), "candidate={} chunk={} not found", candidate_hash, active.chunk_index); diff --git a/core/parachain/validator/impl/parachain_processor.cpp b/core/parachain/validator/impl/parachain_processor.cpp index eb819d6e6f..9bc9a49d54 100644 --- a/core/parachain/validator/impl/parachain_processor.cpp +++ b/core/parachain/validator/impl/parachain_processor.cpp @@ -2027,7 +2027,7 @@ namespace kagome::parachain { rp_state.table_context, rp_state.inject_core_index)) { const auto para_id = backed->candidate.descriptor.para_id; - SL_INFO( + SL_DEBUG( logger_, "Candidate backed.(candidate={}, para id={}, relay_parent={})", summary->candidate, diff --git a/core/parachain/validator/statement_distribution/statement_distribution.cpp b/core/parachain/validator/statement_distribution/statement_distribution.cpp index 52d25e589c..c787e8842c 100644 --- a/core/parachain/validator/statement_distribution/statement_distribution.cpp +++ b/core/parachain/validator/statement_distribution/statement_distribution.cpp @@ -1070,7 +1070,7 @@ namespace kagome::parachain::statement_distribution { [[maybe_unused]] const auto disabled_mask = parachain_state->get().disabled_bitmask(*group); const network::vstaging::AttestedCandidateResponse &response = r.value(); - SL_INFO(logger, + SL_DEBUG(logger, "Fetch attested candidate success. (relay parent={}, " "candidate={}, group index={}, statements={})", relay_parent, diff --git a/housekeeping/docker/kagome-dev/Makefile b/housekeeping/docker/kagome-dev/Makefile index 0c0d985876..6d7365bfe1 100644 --- a/housekeeping/docker/kagome-dev/Makefile +++ b/housekeeping/docker/kagome-dev/Makefile @@ -28,6 +28,8 @@ GITHUB_HUNTER_USERNAME ?= GITHUB_HUNTER_TOKEN ?= CTEST_OUTPUT_ON_FAILURE ?= 1 WERROR ?= OFF +SAN_PARAMS ?= -DCLEAR_OBJS=ON -DCMAKE_TOOLCHAIN_FILE=cmake/toolchain/gcc-13_cxx20.cmake -DASAN=ON +UBSAN_OPTIONS ?= print_stacktrace=1 # Generated versions OS_IMAGE ?= $(OS_IMAGE_NAME):$(OS_IMAGE_TAG)@sha256:$(OS_IMAGE_HASH) @@ -52,6 +54,7 @@ REGION ?= europe-north1 IS_MAIN_OR_TAG ?= false GIT_REF_NAME ?= CI ?= false +BUILD_THREADS=$(shell nproc 2>/dev/null || sysctl -n hw.ncpu) export DOCKER_BUILDKIT=1 # BUILDKIT_PROGRESS - auto, plain, tty, rawjson @@ -76,7 +79,8 @@ kagome_builder: --build-arg LLVM_VERSION=$(LLVM_VERSION) \ --build-arg BASE_IMAGE=$(OS_IMAGE_NAME) \ --build-arg BASE_IMAGE_TAG=$(OS_IMAGE_TAG_WITH_HASH) \ - --build-arg ARCHITECTURE=$(ARCHITECTURE) . + --build-arg ARCHITECTURE=$(ARCHITECTURE) \ + --no-cache . kagome_builder_push: docker push $(DOCKER_REGISTRY_PATH)kagome_builder_deb:$(BUILDER_IMAGE_TAG) ; \ @@ -97,7 +101,6 @@ kagome_dev_docker_build: $(CACHE_DIR)/.cache/ccache ; \ CONTAINER_NAME=kagome_dev_build_$$(openssl rand -hex 6); \ SHORT_COMMIT_HASH=$$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2); \ - BUILD_THREADS=$$(nproc 2>/dev/null || sysctl -n hw.ncpu); \ DOCKER_EXEC_RESULT=0 ; \ echo "Build type: $(BUILD_TYPE)"; \ docker run -d --name $$CONTAINER_NAME \ @@ -105,7 +108,6 @@ kagome_dev_docker_build: --entrypoint "/bin/bash" \ -e SHORT_COMMIT_HASH=$$SHORT_COMMIT_HASH \ -e BUILD_TYPE=$(BUILD_TYPE) \ - -e BUILD_THREADS=$$BUILD_THREADS \ -e PACKAGE_ARCHITECTURE=$(PACKAGE_ARCHITECTURE) \ -e GITHUB_HUNTER_USERNAME=$(GITHUB_HUNTER_USERNAME) \ -e GITHUB_HUNTER_TOKEN=$(GITHUB_HUNTER_TOKEN) \ @@ -126,7 +128,7 @@ kagome_dev_docker_build: git submodule update --init && \ echo \"Building in $$(pwd)\" && \ cmake . -B\"$(BUILD_DIR)\" -G 'Unix Makefiles' -DCMAKE_BUILD_TYPE=\"$(BUILD_TYPE)\" -DBACKWARD=OFF -DWERROR=$(WERROR) && \ - cmake --build \"$(BUILD_DIR)\" --target kagome -- -j${BUILD_THREADS} && \ + cmake --build \"$(BUILD_DIR)\" --target kagome -- -j$(BUILD_THREADS) && \ mkdir -p /tmp/kagome && \ cp /opt/kagome/$(BUILD_DIR)/node/kagome /tmp/kagome/kagome && \ cd /opt/kagome/housekeeping/docker/kagome-dev && \ @@ -154,7 +156,7 @@ kagome_dev_docker_build: fi; \ docker stop $$CONTAINER_NAME -kagome_dev_docker_build_tidy_inhouse: +kagome_dev_docker_build_sanitizers: $(MAKE) get_versions mkdir -p \ $(CACHE_DIR)/.cargo/git \ @@ -163,7 +165,6 @@ kagome_dev_docker_build_tidy_inhouse: $(CACHE_DIR)/.cache/ccache ; \ CONTAINER_NAME=kagome_dev_build_$$(openssl rand -hex 6); \ SHORT_COMMIT_HASH=$$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2); \ - BUILD_THREADS=$$(nproc 2>/dev/null || sysctl -n hw.ncpu); \ DOCKER_EXEC_RESULT=0 ; \ echo "Build type: $(BUILD_TYPE)"; \ docker run -d --name $$CONTAINER_NAME \ @@ -171,18 +172,17 @@ kagome_dev_docker_build_tidy_inhouse: --entrypoint "/bin/bash" \ -e SHORT_COMMIT_HASH=$$SHORT_COMMIT_HASH \ -e BUILD_TYPE=$(BUILD_TYPE) \ - -e BUILD_THREADS=$$BUILD_THREADS \ -e PACKAGE_ARCHITECTURE=$(PACKAGE_ARCHITECTURE) \ -e GITHUB_HUNTER_USERNAME=$(GITHUB_HUNTER_USERNAME) \ -e GITHUB_HUNTER_TOKEN=$(GITHUB_HUNTER_TOKEN) \ -e CTEST_OUTPUT_ON_FAILURE=$(CTEST_OUTPUT_ON_FAILURE) \ + -e UBSAN_OPTIONS=$(UBSAN_OPTIONS) \ -v $$(pwd)/../../../../kagome:/opt/kagome \ - -v $(GOOGLE_APPLICATION_CREDENTIALS):/root/.gcp/google_creds.json \ -v $(CACHE_DIR)/.cargo/git:/root/.cargo/git \ -v $(CACHE_DIR)/.cargo/registry:/root/.cargo/registry \ -v $(CACHE_DIR)/.hunter:/root/.hunter \ -v $(CACHE_DIR)/.cache/ccache:/root/.cache/ccache \ - $(DOCKER_REGISTRY_PATH)kagome_builder_deb:$(BUILDER_IMAGE_TAG) \ + $(DOCKERHUB_BUILDER_PATH):$(BUILDER_IMAGE_TAG) \ -c "tail -f /dev/null"; \ docker exec -t $$CONTAINER_NAME /bin/bash -c \ "clang --version && \ @@ -191,10 +191,10 @@ kagome_dev_docker_build_tidy_inhouse: git config --global --add safe.directory /root/.hunter/_Base/Cache/meta && \ source /venv/bin/activate && \ git submodule update --init && \ - echo \"Building in $$(pwd)\" && \ - cmake . -B\"$(BUILD_DIR)\" -G 'Unix Makefiles' -DCMAKE_BUILD_TYPE=\"$(BUILD_TYPE)\" -DBACKWARD=OFF -DWERROR=$(WERROR) && \ - cmake --build \"$(BUILD_DIR)\" --target generated -- -j${BUILD_THREADS} && \ - cd /opt/kagome/ && export CI='$(CI)' && ./housekeeping/clang-tidy-diff.sh \ + echo \"Building in \`pwd\`\" && \ + cmake . -B\"$(BUILD_DIR)\" -G 'Unix Makefiles' -DBACKWARD=OFF $(SAN_PARAMS) -DWERROR=$(WERROR) && \ + cmake --build build -- -j$(BUILD_THREADS) && \ + cmake --build \"$(BUILD_DIR)\" --target test \ " || DOCKER_EXEC_RESULT=$$? ; \ if [ $$DOCKER_EXEC_RESULT -ne 0 ]; then \ echo "Error: Docker exec failed with return code $$DOCKER_EXEC_RESULT"; \ @@ -212,7 +212,6 @@ kagome_dev_docker_build_tidy: $(CACHE_DIR)/.cache/ccache ; \ CONTAINER_NAME=kagome_dev_build_$$(openssl rand -hex 6); \ SHORT_COMMIT_HASH=$$(grep 'short_commit_hash:' commit_hash.txt | cut -d ' ' -f 2); \ - BUILD_THREADS=$$(nproc 2>/dev/null || sysctl -n hw.ncpu); \ DOCKER_EXEC_RESULT=0 ; \ echo "Build type: $(BUILD_TYPE)"; \ docker run -d --name $$CONTAINER_NAME \ @@ -220,7 +219,6 @@ kagome_dev_docker_build_tidy: --entrypoint "/bin/bash" \ -e SHORT_COMMIT_HASH=$$SHORT_COMMIT_HASH \ -e BUILD_TYPE=$(BUILD_TYPE) \ - -e BUILD_THREADS=$$BUILD_THREADS \ -e PACKAGE_ARCHITECTURE=$(PACKAGE_ARCHITECTURE) \ -e GITHUB_HUNTER_USERNAME=$(GITHUB_HUNTER_USERNAME) \ -e GITHUB_HUNTER_TOKEN=$(GITHUB_HUNTER_TOKEN) \ @@ -241,7 +239,7 @@ kagome_dev_docker_build_tidy: git submodule update --init && \ echo \"Building in $$(pwd)\" && \ cmake . -B\"$(BUILD_DIR)\" -G 'Unix Makefiles' -DCMAKE_BUILD_TYPE=\"$(BUILD_TYPE)\" -DBACKWARD=OFF -DWERROR=$(WERROR) && \ - cmake --build \"$(BUILD_DIR)\" --target generated -- -j${BUILD_THREADS} && \ + cmake --build \"$(BUILD_DIR)\" --target generated -- -j$(BUILD_THREADS) && \ cd /opt/kagome/ && export CI='$(CI)' && ./housekeeping/clang-tidy-diff.sh \ " || DOCKER_EXEC_RESULT=$$? ; \ if [ $$DOCKER_EXEC_RESULT -ne 0 ]; then \ diff --git a/housekeeping/docker/kagome-dev/kagome_builder.Dockerfile b/housekeeping/docker/kagome-dev/kagome_builder.Dockerfile index c91b1dcc68..a248f2384a 100644 --- a/housekeeping/docker/kagome-dev/kagome_builder.Dockerfile +++ b/housekeeping/docker/kagome-dev/kagome_builder.Dockerfile @@ -41,6 +41,7 @@ RUN echo "deb [signed-by=/usr/share/keyrings/llvm-archive-keyring.gpg] http://ap RUN install_packages \ build-essential \ + ninja-build \ ccache \ clang-format-${LLVM_VERSION} \ clang-tidy-${LLVM_VERSION} \ diff --git a/housekeeping/macos/dependency.sh b/housekeeping/macos/dependency.sh index 55148577eb..fd8532d0b5 100755 --- a/housekeeping/macos/dependency.sh +++ b/housekeeping/macos/dependency.sh @@ -15,5 +15,5 @@ sudo python3 -m pip install --upgrade pip sudo python3 -m pip install scikit-build sudo python3 -m pip install cmake==3.25 requests gitpython gcovr -curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain 1.75.0 --profile minimal +curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain 1.81.0 --profile minimal brew install ninja diff --git a/scripts/.env b/scripts/.env index 5dd6309bad..dbc079519e 100644 --- a/scripts/.env +++ b/scripts/.env @@ -1,4 +1,4 @@ -RUST_VERSION=1.77.0 +RUST_VERSION=1.81.0 RUSTUP_HOME=~/.rustup CARGO_HOME=~/.cargo diff --git a/scripts/init.sh b/scripts/init.sh index a4a79d349c..11e3236acd 100755 --- a/scripts/init.sh +++ b/scripts/init.sh @@ -9,7 +9,7 @@ set -a; source $current_dir/.env; set +a #include .env vars apt update apt install --no-install-recommends -y \ build-essential git gcc ca-certificates python-is-python3 python3-pip \ - python3-venv curl libgmp-dev libncurses6 libnsl-dev libseccomp-dev + python3-venv curl libgmp-dev libncurses6 libncurses-dev libnsl-dev libseccomp-dev python3 -m venv "$parent_dir/venv" From ddbf4432795de17ef31ae2cdf8b7319ecc97efc5 Mon Sep 17 00:00:00 2001 From: Ruslan Tushov Date: Fri, 29 Nov 2024 22:33:03 +0500 Subject: [PATCH 06/17] quic (#2298) Signed-off-by: turuslan Signed-off-by: Igor Egorov Co-authored-by: Igor Egorov Co-authored-by: kamilsa --- .../impl/app_configuration_impl.cpp | 86 ++++++------------- .../impl/app_configuration_impl.hpp | 10 --- core/network/impl/peer_manager_impl.cpp | 3 + 3 files changed, 28 insertions(+), 71 deletions(-) diff --git a/core/application/impl/app_configuration_impl.cpp b/core/application/impl/app_configuration_impl.cpp index c19b473cc0..dd5deb2c0a 100644 --- a/core/application/impl/app_configuration_impl.cpp +++ b/core/application/impl/app_configuration_impl.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include "api/transport/tuner.hpp" #include "application/build_version.hpp" @@ -175,11 +176,12 @@ namespace { static constexpr std::array - interpreters{ + interpreters { #if KAGOME_WASM_COMPILER_WASM_EDGE == 1 - "WasmEdge", + "WasmEdge", #endif - "Binaryen"}; + "Binaryen" + }; static const std::string interpreters_str = fmt::format("[{}]", fmt::join(interpreters, ", ")); @@ -606,52 +608,6 @@ namespace kagome::application { return endpoint; } - outcome::result - AppConfigurationImpl::getEndpointFrom( - const libp2p::multi::Multiaddress &multiaddress) const { - using proto = libp2p::multi::Protocol::Code; - constexpr auto NOT_SUPPORTED = std::errc::address_family_not_supported; - constexpr auto BAD_ADDRESS = std::errc::bad_address; - auto host = multiaddress.getFirstValueForProtocol(proto::IP4); - if (not host) { - host = multiaddress.getFirstValueForProtocol(proto::IP6); - } - if (not host) { - SL_ERROR(logger_, - "Address cannot be used to bind to ({}). Only IPv4 and IPv6 " - "interfaces are supported", - multiaddress.getStringAddress()); - return NOT_SUPPORTED; - } - auto port = multiaddress.getFirstValueForProtocol(proto::TCP); - if (not port) { - return NOT_SUPPORTED; - } - uint16_t port_number = 0; - try { - auto wide_port = std::stoul(port.value()); - constexpr auto max_port = std::numeric_limits::max(); - if (wide_port > max_port or 0 == wide_port) { - SL_ERROR( - logger_, - "Port value ({}) cannot be zero or greater than {} (address {})", - wide_port, - max_port, - multiaddress.getStringAddress()); - return BAD_ADDRESS; - } - port_number = static_cast(wide_port); - } catch (...) { - // only std::out_of_range or std::invalid_argument are possible - SL_ERROR(logger_, - "Passed value {} is not a valid port number within address {}", - port.value(), - multiaddress.getStringAddress()); - return BAD_ADDRESS; - } - return getEndpointFrom(host.value(), port_number); - } - bool AppConfigurationImpl::testListenAddresses() const { auto temp_context = std::make_shared(); constexpr auto kZeroPortTolerance = 0; @@ -664,23 +620,31 @@ namespace kagome::application { addr.getStringAddress()); return false; } - auto endpoint = getEndpointFrom(addr); - if (not endpoint) { + auto tcp = libp2p::transport::detail::asTcp(addr); + auto quic = libp2p::transport::detail::asQuic(addr); + if (not(tcp and tcp.value().first.asTcp()) + and not(quic and quic.value().asUdp())) { SL_ERROR(logger_, "Endpoint cannot be constructed from address {}", addr.getStringAddress()); return false; } - try { - boost::system::error_code error_code; - auto acceptor = api::acceptOnFreePort( - temp_context, endpoint.value(), kZeroPortTolerance, logger_); - acceptor->cancel(error_code); - acceptor->close(error_code); - } catch (...) { - SL_ERROR( - logger_, "Unable to listen on address {}", addr.getStringAddress()); - return false; + if (tcp) { + try { + boost::system::error_code error_code; + auto acceptor = + api::acceptOnFreePort(temp_context, + tcp.value().first.asTcp().value(), + kZeroPortTolerance, + logger_); + acceptor->cancel(error_code); + acceptor->close(error_code); + } catch (...) { + SL_ERROR(logger_, + "Unable to listen on address {}", + addr.getStringAddress()); + return false; + } } } return true; diff --git a/core/application/impl/app_configuration_impl.hpp b/core/application/impl/app_configuration_impl.hpp index e87e9149d9..2421b9f874 100644 --- a/core/application/impl/app_configuration_impl.hpp +++ b/core/application/impl/app_configuration_impl.hpp @@ -301,16 +301,6 @@ namespace kagome::application { boost::asio::ip::tcp::endpoint getEndpointFrom(const std::string &host, uint16_t port) const; - /** - * Convert a given libp2p multiaddress into a boost tcp::endpoint format. - * @param multiaddress - an address to be converted. Should contain a valid - * interface name or IP4/IP6 address and a port value to listen on. - * @return boost tcp::endpoint when well-formed multiaddress is passed, - * otherwise - an error - */ - outcome::result getEndpointFrom( - const libp2p::multi::Multiaddress &multiaddress) const; - /** * Checks whether configured listen addresses are available. * @return true when addresses are available, false - when at least one diff --git a/core/network/impl/peer_manager_impl.cpp b/core/network/impl/peer_manager_impl.cpp index 1e6224013e..53cf6f7caf 100644 --- a/core/network/impl/peer_manager_impl.cpp +++ b/core/network/impl/peer_manager_impl.cpp @@ -191,6 +191,9 @@ namespace kagome::network { // Enqueue bootstrap nodes with permanent lifetime for (const auto &bootstrap_node : bootstrap_nodes_) { + if (own_peer_info_.id == bootstrap_node.id) { + continue; + } kademlia_->addPeer(bootstrap_node, true); } From d6c01fa88f7383fad12fac6c6c7ba7e8bce09d48 Mon Sep 17 00:00:00 2001 From: Boris <31869190+ErakhtinB@users.noreply.github.com> Date: Sat, 30 Nov 2024 15:19:54 +0500 Subject: [PATCH 07/17] Sanitizer fix (#2297) --- .../provisioner/impl/prioritized_selection.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/dispute_coordinator/provisioner/impl/prioritized_selection.cpp b/core/dispute_coordinator/provisioner/impl/prioritized_selection.cpp index e455016fdc..57d823e7d4 100644 --- a/core/dispute_coordinator/provisioner/impl/prioritized_selection.cpp +++ b/core/dispute_coordinator/provisioner/impl/prioritized_selection.cpp @@ -71,15 +71,17 @@ namespace kagome::dispute { // onchain - don't skip it. In this case we'd better push as much fresh // votes as possible to bring it to conclusion faster. // https://github.com/paritytech/polkadot/blob/40974fb99c86f5c341105b7db53c7aa0df707d66/node/core/provisioner/src/disputes/prioritized_selection/mod.rs#L146 - for (auto &recent_dispute : std::move(recent_disputes)) { + decltype(recent_disputes) filtered_disputes; + for (auto &recent_dispute : recent_disputes) { auto &dispute_status = std::get<2>(recent_dispute); if (is_type(dispute_status) // is_confirmed_concluded or onchain.contains(std::tie(std::get<0>(recent_dispute), std::get<1>(recent_dispute)))) { - recent_disputes.emplace_back(std::move(recent_dispute)); + filtered_disputes.emplace_back(std::move(recent_dispute)); } } + recent_disputes = std::move(filtered_disputes); auto partitioned = partition_recent_disputes(recent_disputes, onchain); From 2a9083b3caf3f3cd1c61a8ceb908595ac87d444d Mon Sep 17 00:00:00 2001 From: Harrm Date: Mon, 2 Dec 2024 16:20:08 +0300 Subject: [PATCH 08/17] Feature/trie values column (#2288) * Separate trie nodes and values and fix batch processing * Mark all operator bool's explicit and fix ValueAndHash operator== * Fix base rocksdb test not deleting ttl_migrated * Link in_memory_storage to core_integration_test --- core/application/chain_spec.hpp | 4 +- .../impl/kagome_application_impl.cpp | 18 +- core/blockchain/indexer.hpp | 4 +- core/common/CMakeLists.txt | 2 +- core/common/blob.hpp | 8 +- .../babe/has_babe_consensus_digest.hpp | 2 +- .../babe/impl/babe_config_repository_impl.hpp | 2 +- .../grandpa/has_authority_set_change.hpp | 2 +- .../grandpa/impl/authority_manager_impl.hpp | 3 +- core/consensus/timeline/types.hpp | 10 +- .../impl/dispute_coordinator_impl.cpp | 12 +- core/host_api/host_api.hpp | 7 +- core/injector/application_injector.cpp | 8 +- core/injector/application_injector.hpp | 3 + .../impl/protocols/beefy_protocol_impl.cpp | 4 +- .../protocols/fetch_attested_candidate.hpp | 22 +-- core/network/impl/protocols/parachain.cpp | 2 +- .../impl/protocols/protocol_fetch_chunk.hpp | 18 +- .../protocol_fetch_chunk_obsolete.hpp | 20 ++- .../impl/protocols/protocol_req_collation.cpp | 6 +- .../impl/protocols/protocol_req_pov.cpp | 20 +-- .../impl/protocols/send_dispute_protocol.hpp | 19 +- core/network/impl/router_libp2p.cpp | 3 +- core/network/impl/state_sync_request_flow.cpp | 71 +++++--- core/network/impl/state_sync_request_flow.hpp | 9 +- core/network/impl/synchronizer_impl.cpp | 9 +- .../approval/approval_distribution.hpp | 5 +- .../statement_distribution.cpp | 6 +- core/primitives/event_types.hpp | 2 +- .../runtime/common/module_repository_impl.cpp | 5 +- .../common/trie_storage_provider_impl.cpp | 2 +- core/storage/CMakeLists.txt | 3 +- core/storage/buffer_map_types.hpp | 5 +- core/storage/face/batch_writeable.hpp | 5 +- core/storage/face/generic_maps.hpp | 12 +- core/storage/face/write_batch.hpp | 39 +++++ core/storage/in_memory/in_memory_batch.hpp | 47 ----- core/storage/map_prefix/prefix.cpp | 4 +- core/storage/map_prefix/prefix.hpp | 7 +- core/storage/migrations/migrations.cpp | 164 ++++++++++++++++++ core/storage/migrations/migrations.hpp | 34 ++++ core/storage/rocksdb/rocksdb.cpp | 157 +++++++++-------- core/storage/rocksdb/rocksdb.hpp | 32 ++-- core/storage/rocksdb/rocksdb_batch.cpp | 39 +++-- core/storage/rocksdb/rocksdb_batch.hpp | 19 +- core/storage/rocksdb/rocksdb_spaces.cpp | 38 ++-- core/storage/rocksdb/rocksdb_spaces.hpp | 10 +- core/storage/rocksdb/rocksdb_util.hpp | 11 ++ core/storage/spaced_storage.hpp | 8 +- core/storage/trie/child_prefix.hpp | 2 +- core/storage/trie/compact_decode.cpp | 16 +- core/storage/trie/compact_decode.hpp | 1 + core/storage/trie/compact_encode.cpp | 35 +++- .../trie/impl/topper_trie_batch_impl.cpp | 21 ++- .../trie/impl/topper_trie_batch_impl.hpp | 10 +- .../trie/impl/trie_storage_backend_batch.cpp | 35 ---- .../trie/impl/trie_storage_backend_batch.hpp | 34 ---- .../trie/impl/trie_storage_backend_impl.cpp | 43 +---- .../trie/impl/trie_storage_backend_impl.hpp | 27 +-- .../polkadot_trie/polkadot_trie_cursor.hpp | 8 + .../polkadot_trie_cursor_impl.cpp | 26 +++ .../polkadot_trie_cursor_impl.hpp | 4 + .../trie/polkadot_trie/polkadot_trie_impl.hpp | 1 + core/storage/trie/polkadot_trie/trie_node.hpp | 6 +- core/storage/trie/raw_cursor.hpp | 43 +++-- .../serialization/trie_serializer_impl.cpp | 55 +++--- .../serialization/trie_serializer_impl.hpp | 4 +- core/storage/trie/trie_batches.hpp | 2 + core/storage/trie/trie_storage_backend.hpp | 11 +- .../trie_pruner/impl/trie_pruner_impl.cpp | 38 ++-- .../trie_pruner/impl/trie_pruner_impl.hpp | 6 +- core/utils/kagome_db_editor.cpp | 45 +++-- core/utils/map_entry.hpp | 2 +- test/core/consensus/CMakeLists.txt | 1 + test/core/consensus/beefy_test.cpp | 2 +- test/core/consensus/grandpa/CMakeLists.txt | 1 + .../grandpa/authority_manager_test.cpp | 2 +- test/core/consensus/timeline/CMakeLists.txt | 1 + .../consensus/timeline/slots_util_test.cpp | 2 +- test/core/network/CMakeLists.txt | 1 + .../network/state_protocol_observer_test.cpp | 2 +- test/core/runtime/CMakeLists.txt | 2 + test/core/runtime/binaryen/CMakeLists.txt | 4 + test/core/runtime/runtime_test_base.hpp | 2 +- .../runtime/runtime_upgrade_tracker_test.cpp | 5 +- .../runtime/trie_storage_provider_test.cpp | 2 +- test/core/runtime/wavm/CMakeLists.txt | 1 + test/core/storage/changes_trie/CMakeLists.txt | 1 + .../changes_trie/changes_tracker_test.cpp | 4 +- .../trie/polkadot_trie/polkadot_trie_test.cpp | 2 +- .../trie/polkadot_trie_cursor_dummy.hpp | 9 +- .../storage/trie/trie_storage/CMakeLists.txt | 9 +- .../polkadot_codec_node_decoding_test.cpp | 3 +- .../trie/trie_storage/trie_batch_test.cpp | 10 +- .../trie_storage_backend_test.cpp | 93 ---------- .../storage/trie_pruner/trie_pruner_test.cpp | 114 ++++++------ test/external-project-test/src/main.cpp | 8 +- .../core/storage/generic_storage_mock.hpp | 7 +- .../mock/core/storage/spaced_storage_mock.hpp | 10 +- .../storage/trie/polkadot_trie_cursor_mock.h | 5 + .../trie/trie_storage_backend_mock.hpp | 34 +--- test/mock/core/storage/write_batch_mock.hpp | 21 +++ test/testutil/storage/CMakeLists.txt | 7 + test/testutil/storage/base_rocksdb_test.cpp | 2 +- test/testutil/storage/base_rocksdb_test.hpp | 3 +- .../testutil}/storage/in_memory/cursor.hpp | 2 +- .../storage/in_memory/in_memory_batch.hpp | 84 +++++++++ .../in_memory/in_memory_spaced_storage.hpp | 8 +- .../storage/in_memory/in_memory_storage.cpp | 10 +- .../storage/in_memory/in_memory_storage.hpp | 4 +- 110 files changed, 1139 insertions(+), 756 deletions(-) delete mode 100644 core/storage/in_memory/in_memory_batch.hpp create mode 100644 core/storage/migrations/migrations.cpp create mode 100644 core/storage/migrations/migrations.hpp delete mode 100644 core/storage/trie/impl/trie_storage_backend_batch.cpp delete mode 100644 core/storage/trie/impl/trie_storage_backend_batch.hpp delete mode 100644 test/core/storage/trie/trie_storage/trie_storage_backend_test.cpp rename {core => test/testutil}/storage/in_memory/cursor.hpp (96%) create mode 100644 test/testutil/storage/in_memory/in_memory_batch.hpp rename {core => test/testutil}/storage/in_memory/in_memory_spaced_storage.hpp (76%) rename {core => test/testutil}/storage/in_memory/in_memory_storage.cpp (86%) rename {core => test/testutil}/storage/in_memory/in_memory_storage.hpp (90%) diff --git a/core/application/chain_spec.hpp b/core/application/chain_spec.hpp index 73c24bb0e7..facd590cde 100644 --- a/core/application/chain_spec.hpp +++ b/core/application/chain_spec.hpp @@ -38,8 +38,8 @@ namespace kagome::application { virtual const std::vector &bootNodes() const = 0; - virtual const std::vector> - &telemetryEndpoints() const = 0; + virtual const std::vector> & + telemetryEndpoints() const = 0; virtual const std::string &protocolId() const = 0; diff --git a/core/application/impl/kagome_application_impl.cpp b/core/application/impl/kagome_application_impl.cpp index eaab11dd0b..bbfd896221 100644 --- a/core/application/impl/kagome_application_impl.cpp +++ b/core/application/impl/kagome_application_impl.cpp @@ -6,6 +6,7 @@ #include "application/impl/kagome_application_impl.hpp" +#include #include #include @@ -17,6 +18,7 @@ #include "injector/application_injector.hpp" #include "metrics/metrics.hpp" #include "parachain/pvf/secure_mode_precheck.hpp" +#include "storage/migrations/migrations.hpp" #include "telemetry/service.hpp" #include "utils/watchdog.hpp" @@ -70,10 +72,11 @@ namespace kagome::application { getpid()); auto chain_path = app_config_->chainPath(chain_spec_->id()); - auto storage_backend = app_config_->storageBackend() - == AppConfiguration::StorageBackend::RocksDB - ? "RocksDB" - : "Unknown"; + const char *storage_backend = + app_config_->storageBackend() + == AppConfiguration::StorageBackend::RocksDB + ? "RocksDB" + : "Unknown"; logger_->info("Chain path is {}, storage backend is {}", chain_path.native(), storage_backend); @@ -144,6 +147,13 @@ namespace kagome::application { "platform. Proceed at your own risk."); #endif + if (app_config_->enableDbMigration()) { + if (auto res = storage::migrations::runMigrations(injector_); !res) { + SL_ERROR(logger_, "Failed to migrate the database: {}", res.error()); + exit(EXIT_FAILURE); + } + } + app_state_manager->run(); watchdog->stop(); diff --git a/core/blockchain/indexer.hpp b/core/blockchain/indexer.hpp index 428b047c77..c02566f9f3 100644 --- a/core/blockchain/indexer.hpp +++ b/core/blockchain/indexer.hpp @@ -94,7 +94,7 @@ namespace kagome::blockchain { */ template struct Indexer { - Indexer(std::shared_ptr db, + Indexer(std::shared_ptr db, std::shared_ptr block_tree) : db_{std::move(db)}, block_tree_{std::move(block_tree)} { primitives::BlockInfo genesis{0, block_tree_->getGenesisBlockHash()}; @@ -252,7 +252,7 @@ namespace kagome::blockchain { return raw->kv; } - std::shared_ptr db_; + std::shared_ptr db_; std::shared_ptr block_tree_; primitives::BlockInfo last_finalized_indexed_; std::map> map_; diff --git a/core/common/CMakeLists.txt b/core/common/CMakeLists.txt index ba231b80bc..f167b02b6b 100644 --- a/core/common/CMakeLists.txt +++ b/core/common/CMakeLists.txt @@ -9,7 +9,7 @@ target_link_libraries(hexutil Boost::boost outcome) kagome_install(hexutil) add_library(blob blob.hpp blob.cpp) -target_link_libraries(blob hexutil) +target_link_libraries(blob hexutil scale::scale) kagome_install(blob) add_library(fd_limit fd_limit.hpp fd_limit.cpp) diff --git a/core/common/blob.hpp b/core/common/blob.hpp index 87ec763881..1a0807c5ec 100644 --- a/core/common/blob.hpp +++ b/core/common/blob.hpp @@ -90,8 +90,8 @@ struct fmt::formatter \ : fmt::formatter { \ template \ - auto format(const space_name::class_name &blob, FormatCtx &ctx) const \ - -> decltype(ctx.out()) { \ + auto format(const space_name::class_name &blob, \ + FormatCtx &ctx) const -> decltype(ctx.out()) { \ return fmt::formatter::format(blob, ctx); \ } \ }; @@ -277,8 +277,8 @@ struct fmt::formatter> { // Formats the Blob using the parsed format specification (presentation) // stored in this formatter. template - auto format(const kagome::common::Blob &blob, FormatContext &ctx) const - -> decltype(ctx.out()) { + auto format(const kagome::common::Blob &blob, + FormatContext &ctx) const -> decltype(ctx.out()) { if (presentation == 's') { if constexpr (N > 4) { uint16_t head = static_cast(blob[1]) diff --git a/core/consensus/babe/has_babe_consensus_digest.hpp b/core/consensus/babe/has_babe_consensus_digest.hpp index cd8f61ba75..21c8b3e35a 100644 --- a/core/consensus/babe/has_babe_consensus_digest.hpp +++ b/core/consensus/babe/has_babe_consensus_digest.hpp @@ -48,7 +48,7 @@ namespace kagome::consensus::babe { } } - operator bool() const { + explicit operator bool() const { return epoch.has_value(); } diff --git a/core/consensus/babe/impl/babe_config_repository_impl.hpp b/core/consensus/babe/impl/babe_config_repository_impl.hpp index ec1d2bc1b9..595678f4e6 100644 --- a/core/consensus/babe/impl/babe_config_repository_impl.hpp +++ b/core/consensus/babe/impl/babe_config_repository_impl.hpp @@ -122,7 +122,7 @@ namespace kagome::consensus::babe { void warp(Indexer &indexer_, const primitives::BlockInfo &block); - std::shared_ptr persistent_storage_; + std::shared_ptr persistent_storage_; bool config_warp_sync_; EpochTimings &timings_; std::shared_ptr block_tree_; diff --git a/core/consensus/grandpa/has_authority_set_change.hpp b/core/consensus/grandpa/has_authority_set_change.hpp index e9e0fcaa6e..ac63f149dd 100644 --- a/core/consensus/grandpa/has_authority_set_change.hpp +++ b/core/consensus/grandpa/has_authority_set_change.hpp @@ -36,7 +36,7 @@ namespace kagome::consensus::grandpa { } } - operator bool() const { + explicit operator bool() const { return scheduled || forced; } diff --git a/core/consensus/grandpa/impl/authority_manager_impl.hpp b/core/consensus/grandpa/impl/authority_manager_impl.hpp index 3e6768dd64..0657a223ae 100644 --- a/core/consensus/grandpa/impl/authority_manager_impl.hpp +++ b/core/consensus/grandpa/impl/authority_manager_impl.hpp @@ -14,6 +14,7 @@ #include "consensus/grandpa/has_authority_set_change.hpp" #include "log/logger.hpp" #include "primitives/event_types.hpp" +#include "storage/buffer_map_types.hpp" #include "storage/spaced_storage.hpp" namespace kagome::application { @@ -95,7 +96,7 @@ namespace kagome::consensus::grandpa { std::shared_ptr block_tree_; std::shared_ptr grandpa_api_; - std::shared_ptr persistent_storage_; + std::shared_ptr persistent_storage_; primitives::events::ChainSub chain_sub_; mutable blockchain::Indexer indexer_; diff --git a/core/consensus/timeline/types.hpp b/core/consensus/timeline/types.hpp index 616b1fadb7..4cce3c3203 100644 --- a/core/consensus/timeline/types.hpp +++ b/core/consensus/timeline/types.hpp @@ -34,7 +34,8 @@ namespace kagome::consensus { template SlotDuration(const std::chrono::duration &duration) : std::chrono::milliseconds( - std::chrono::duration_cast(duration)) {} + std::chrono::duration_cast(duration)) { + } template requires std::is_integral_v @@ -47,7 +48,7 @@ namespace kagome::consensus { } // Convert to boolean - operator bool() const { + explicit operator bool() const { return count() != 0; } @@ -95,8 +96,9 @@ namespace kagome::consensus { /// Epoch length in slots EpochLength epoch_length{0}; - operator bool() const { - return (bool)slot_duration and (bool) epoch_length; + explicit operator bool() const { + return static_cast(slot_duration) + and static_cast(epoch_length); } void init(SlotDuration _slot_duration, EpochLength _epoch_length) { diff --git a/core/dispute_coordinator/impl/dispute_coordinator_impl.cpp b/core/dispute_coordinator/impl/dispute_coordinator_impl.cpp index 9ce629399d..3eec26b0cd 100644 --- a/core/dispute_coordinator/impl/dispute_coordinator_impl.cpp +++ b/core/dispute_coordinator/impl/dispute_coordinator_impl.cpp @@ -1232,12 +1232,12 @@ namespace kagome::dispute { auto is_old_concluded_for = intermediate_result.old_state.dispute_status.has_value() ? is_type( - intermediate_result.old_state.dispute_status.value()) + intermediate_result.old_state.dispute_status.value()) : false; auto is_new_concluded_for = intermediate_result.new_state.dispute_status.has_value() ? is_type( - intermediate_result.new_state.dispute_status.value()) + intermediate_result.new_state.dispute_status.value()) : false; auto is_freshly_concluded_for = not is_old_concluded_for and is_new_concluded_for; @@ -1245,12 +1245,12 @@ namespace kagome::dispute { auto is_old_concluded_against = intermediate_result.old_state.dispute_status.has_value() ? is_type( - intermediate_result.old_state.dispute_status.value()) + intermediate_result.old_state.dispute_status.value()) : false; auto is_new_concluded_against = intermediate_result.new_state.dispute_status.has_value() ? is_type( - intermediate_result.new_state.dispute_status.value()) + intermediate_result.new_state.dispute_status.value()) : false; auto is_freshly_concluded_against = not is_old_concluded_against and is_new_concluded_against; @@ -1261,12 +1261,12 @@ namespace kagome::dispute { auto is_old_confirmed_concluded = intermediate_result.old_state.dispute_status.has_value() ? not is_type( - intermediate_result.old_state.dispute_status.value()) + intermediate_result.old_state.dispute_status.value()) : false; auto is_new_confirmed_concluded = intermediate_result.new_state.dispute_status.has_value() ? not is_type( - intermediate_result.new_state.dispute_status.value()) + intermediate_result.new_state.dispute_status.value()) : false; auto is_freshly_confirmed = not is_old_confirmed_concluded and is_new_confirmed_concluded; diff --git a/core/host_api/host_api.hpp b/core/host_api/host_api.hpp index 093f073277..0a975cccf3 100644 --- a/core/host_api/host_api.hpp +++ b/core/host_api/host_api.hpp @@ -411,9 +411,10 @@ namespace kagome::host_api { * store. */ [[nodiscard]] virtual runtime::WasmSpan - ext_crypto_ecdsa_sign_prehashed_version_1(runtime::WasmSize key_type, - runtime::WasmPointer key, - runtime::WasmPointer msg_data) = 0; + ext_crypto_ecdsa_sign_prehashed_version_1( + runtime::WasmSize key_type, + runtime::WasmPointer key, + runtime::WasmPointer msg_data) = 0; /** * @brief Generates an ecdsa key for the given key type using an optional diff --git a/core/injector/application_injector.cpp b/core/injector/application_injector.cpp index 1ea840fb79..23c49e26cd 100644 --- a/core/injector/application_injector.cpp +++ b/core/injector/application_injector.cpp @@ -939,7 +939,7 @@ namespace kagome::injector { KagomeNodeInjector::KagomeNodeInjector( sptr app_config) : pimpl_{std::make_unique( - makeKagomeNodeInjector(std::move(app_config)))} {} + makeKagomeNodeInjector(std::move(app_config)))} {} sptr KagomeNodeInjector::injectAppConfig() { return pimpl_->injector_ @@ -1126,6 +1126,12 @@ namespace kagome::injector { return pimpl_->injector_.template create>(); } + std::shared_ptr + KagomeNodeInjector::injectRuntimeUpgradeTracker() { + return pimpl_->injector_ + .template create>(); + } + void KagomeNodeInjector::kademliaRandomWalk() { pimpl_->injector_.create>(); } diff --git a/core/injector/application_injector.hpp b/core/injector/application_injector.hpp index e891b8a64c..ea68b4ebfe 100644 --- a/core/injector/application_injector.hpp +++ b/core/injector/application_injector.hpp @@ -12,6 +12,7 @@ #include "clock/clock.hpp" #include "network/dispute_request_observer.hpp" +#include "runtime/runtime_upgrade_tracker.hpp" #include "storage/spaced_storage.hpp" namespace soralog { @@ -152,6 +153,8 @@ namespace kagome::injector { std::shared_ptr injectStorage(); std::shared_ptr injectAddressPublisher(); + std::shared_ptr + injectRuntimeUpgradeTracker(); void kademliaRandomWalk(); std::shared_ptr diff --git a/core/network/impl/protocols/beefy_protocol_impl.cpp b/core/network/impl/protocols/beefy_protocol_impl.cpp index 5e8d8846b2..915f694d18 100644 --- a/core/network/impl/protocols/beefy_protocol_impl.cpp +++ b/core/network/impl/protocols/beefy_protocol_impl.cpp @@ -23,7 +23,9 @@ namespace kagome::network { Roles roles, std::shared_ptr beefy) : notifications_{notifications_factory.make( - {make_protocols(kBeefyProtocol, genesis)}, kPeersLimit, kPeersLimit)}, + {make_protocols(kBeefyProtocol, genesis)}, + kPeersLimit, + kPeersLimit)}, roles_{roles}, beefy_{std::move(beefy)} {} diff --git a/core/network/impl/protocols/fetch_attested_candidate.hpp b/core/network/impl/protocols/fetch_attested_candidate.hpp index d4f2bd95e7..2ec93b0525 100644 --- a/core/network/impl/protocols/fetch_attested_candidate.hpp +++ b/core/network/impl/protocols/fetch_attested_candidate.hpp @@ -38,17 +38,17 @@ namespace kagome::network { parachain::statement_distribution::StatementDistribution> statement_distribution) : RequestResponseProtocolImpl< - vstaging::AttestedCandidateRequest, - vstaging::AttestedCandidateResponse, - ScaleMessageReadWriter>{kFetchAttestedCandidateProtocolName, - host, - make_protocols( - kFetchAttestedCandidateProtocol, - genesis_hash, - kProtocolPrefixPolkadot), - log::createLogger( - kFetchAttestedCandidateProtocolName, - "req_attested_candidate_protocol")}, + vstaging::AttestedCandidateRequest, + vstaging::AttestedCandidateResponse, + ScaleMessageReadWriter>{kFetchAttestedCandidateProtocolName, + host, + make_protocols( + kFetchAttestedCandidateProtocol, + genesis_hash, + kProtocolPrefixPolkadot), + log::createLogger( + kFetchAttestedCandidateProtocolName, + "req_attested_candidate_protocol")}, statement_distribution_(std::move(statement_distribution)) { BOOST_ASSERT(statement_distribution_); } diff --git a/core/network/impl/protocols/parachain.cpp b/core/network/impl/protocols/parachain.cpp index 1d703109e7..a35a092a75 100644 --- a/core/network/impl/protocols/parachain.cpp +++ b/core/network/impl/protocols/parachain.cpp @@ -66,7 +66,7 @@ namespace kagome::network { size_t limit_in, size_t limit_out) : notifications_{inject.notifications_factory->make( - std::move(protocols_groups), limit_in, limit_out)}, + std::move(protocols_groups), limit_in, limit_out)}, collation_versions_{CollationVersion::VStaging, CollationVersion::V1}, roles_{inject.roles}, peer_manager_{inject.peer_manager}, diff --git a/core/network/impl/protocols/protocol_fetch_chunk.hpp b/core/network/impl/protocols/protocol_fetch_chunk.hpp index 183e4c7c92..61fbf4440e 100644 --- a/core/network/impl/protocols/protocol_fetch_chunk.hpp +++ b/core/network/impl/protocols/protocol_fetch_chunk.hpp @@ -45,15 +45,15 @@ namespace kagome::network { std::shared_ptr pp, std::shared_ptr pm) : RequestResponseProtocolImpl< - FetchChunkRequest, - FetchChunkResponse, - ScaleMessageReadWriter>{kFetchChunkProtocolName, - host, - make_protocols(kFetchChunkProtocol, - genesis_hash, - kProtocolPrefixPolkadot), - log::createLogger(kFetchChunkProtocolName, - "req_chunk_protocol")}, + FetchChunkRequest, + FetchChunkResponse, + ScaleMessageReadWriter>{kFetchChunkProtocolName, + host, + make_protocols(kFetchChunkProtocol, + genesis_hash, + kProtocolPrefixPolkadot), + log::createLogger(kFetchChunkProtocolName, + "req_chunk_protocol")}, pp_{std::move(pp)}, pm_{std::move(pm)} { BOOST_ASSERT(pp_); diff --git a/core/network/impl/protocols/protocol_fetch_chunk_obsolete.hpp b/core/network/impl/protocols/protocol_fetch_chunk_obsolete.hpp index 879a538dd3..5ba8a6ec77 100644 --- a/core/network/impl/protocols/protocol_fetch_chunk_obsolete.hpp +++ b/core/network/impl/protocols/protocol_fetch_chunk_obsolete.hpp @@ -16,6 +16,7 @@ #include "blockchain/genesis_block_hash.hpp" #include "log/logger.hpp" #include "network/common.hpp" +#include "network/helpers/scale_message_read_writer.hpp" #include "network/impl/protocols/request_response_protocol.hpp" #include "parachain/validator/parachain_processor.hpp" #include "utils/non_copyable.hpp" @@ -44,15 +45,16 @@ namespace kagome::network { const blockchain::GenesisBlockHash &genesis_hash, std::shared_ptr pp) : RequestResponseProtocolImpl< - FetchChunkRequest, - FetchChunkResponseObsolete, - ScaleMessageReadWriter>{kFetchChunkProtocolName, - host, - make_protocols(kFetchChunkProtocolObsolete, - genesis_hash, - kProtocolPrefixPolkadot), - log::createLogger(kFetchChunkProtocolName, - "req_chunk_protocol")}, + FetchChunkRequest, + FetchChunkResponseObsolete, + ScaleMessageReadWriter>{kFetchChunkProtocolName, + host, + make_protocols( + kFetchChunkProtocolObsolete, + genesis_hash, + kProtocolPrefixPolkadot), + log::createLogger(kFetchChunkProtocolName, + "req_chunk_protocol")}, pp_{std::move(pp)} { BOOST_ASSERT(pp_); } diff --git a/core/network/impl/protocols/protocol_req_collation.cpp b/core/network/impl/protocols/protocol_req_collation.cpp index 84c0694e60..a2663d3ca6 100644 --- a/core/network/impl/protocols/protocol_req_collation.cpp +++ b/core/network/impl/protocols/protocol_req_collation.cpp @@ -62,9 +62,9 @@ namespace kagome::network { const blockchain::GenesisBlockHash &genesis_hash, std::shared_ptr observer) : v1_impl_{std::make_shared< - ReqCollationProtocolImpl>( - host, kReqCollationProtocol, chain_spec, genesis_hash, observer)}, + ReqCollationProtocolImpl>( + host, kReqCollationProtocol, chain_spec, genesis_hash, observer)}, vstaging_impl_{std::make_shared< ReqCollationProtocolImpl>( diff --git a/core/network/impl/protocols/protocol_req_pov.cpp b/core/network/impl/protocols/protocol_req_pov.cpp index d7eee420ee..75a97d1b91 100644 --- a/core/network/impl/protocols/protocol_req_pov.cpp +++ b/core/network/impl/protocols/protocol_req_pov.cpp @@ -25,15 +25,15 @@ namespace kagome::network { const blockchain::GenesisBlockHash &genesis_hash, std::shared_ptr observer) : RequestResponseProtocolImpl< - RequestPov, - ResponsePov, - ScaleMessageReadWriter>{kReqPovProtocolName, - host, - make_protocols(kReqPovProtocol, - genesis_hash, - kProtocolPrefixPolkadot), - log::createLogger(kReqPovProtocolName, - "req_pov_protocol")}, + RequestPov, + ResponsePov, + ScaleMessageReadWriter>{kReqPovProtocolName, + host, + make_protocols(kReqPovProtocol, + genesis_hash, + kProtocolPrefixPolkadot), + log::createLogger(kReqPovProtocolName, + "req_pov_protocol")}, observer_{std::move(observer)} {} protected: @@ -73,7 +73,7 @@ namespace kagome::network { const blockchain::GenesisBlockHash &genesis_hash, std::shared_ptr observer) : impl_{std::make_shared( - host, chain_spec, genesis_hash, std::move(observer))} {} + host, chain_spec, genesis_hash, std::move(observer))} {} const Protocol &ReqPovProtocol::protocolName() const { BOOST_ASSERT(impl_ && !!"ReqPovProtocolImpl must be initialized!"); diff --git a/core/network/impl/protocols/send_dispute_protocol.hpp b/core/network/impl/protocols/send_dispute_protocol.hpp index aca04a9138..e09f3e7f64 100644 --- a/core/network/impl/protocols/send_dispute_protocol.hpp +++ b/core/network/impl/protocols/send_dispute_protocol.hpp @@ -47,15 +47,16 @@ namespace kagome::network { std::shared_ptr dispute_request_observer) : RequestResponseProtocolImpl< - DisputeRequest, - DisputeResponse, - ScaleMessageReadWriter>{kSendDisputeProtocolName, - host, - make_protocols(kSendDisputeProtocol, - genesis_hash, - kProtocolPrefixPolkadot), - log::createLogger(kSendDisputeProtocolName, - "dispute_protocol")}, + DisputeRequest, + DisputeResponse, + ScaleMessageReadWriter>{kSendDisputeProtocolName, + host, + make_protocols(kSendDisputeProtocol, + genesis_hash, + kProtocolPrefixPolkadot), + log::createLogger( + kSendDisputeProtocolName, + "dispute_protocol")}, dispute_request_observer_{std::move(dispute_request_observer)} { BOOST_ASSERT(dispute_request_observer_); } diff --git a/core/network/impl/router_libp2p.cpp b/core/network/impl/router_libp2p.cpp index 7ca309191c..c79fd4d2f2 100644 --- a/core/network/impl/router_libp2p.cpp +++ b/core/network/impl/router_libp2p.cpp @@ -8,6 +8,7 @@ #include #include + #include "common/main_thread_pool.hpp" #include "network/impl/protocols/beefy_justification_protocol.hpp" #include "network/impl/protocols/beefy_protocol_impl.hpp" @@ -179,7 +180,7 @@ namespace kagome::network { auto &addr_repo = host_.getPeerRepository().getAddressRepository(); // here we put our known public addresses to the repository auto upsert_res = addr_repo.upsertAddresses( - own_info_.id, own_info_.addresses, libp2p::peer::ttl::kPermanent); + own_info_.id, own_info_.addresses, std::chrono::years(100)); if (!upsert_res) { log_->error("Cannot add own addresses to repo: {}", upsert_res.error()); } diff --git a/core/network/impl/state_sync_request_flow.cpp b/core/network/impl/state_sync_request_flow.cpp index 1a62d750c2..b76d99682a 100644 --- a/core/network/impl/state_sync_request_flow.cpp +++ b/core/network/impl/state_sync_request_flow.cpp @@ -14,6 +14,22 @@ #include "storage/trie/trie_storage_backend.hpp" namespace kagome::network { + + outcome::result> + StateSyncRequestFlow::create( + std::shared_ptr node_db, + const primitives::BlockInfo &block_info, + const primitives::BlockHeader &block) { + std::unique_ptr flow{ + new StateSyncRequestFlow(node_db, block_info, block)}; + OUTCOME_TRY(done, flow->isKnown(block.state_root)); + flow->done_ = done; + if (not done) { + flow->levels_.emplace_back(Level{.branch_hash = block.state_root}); + } + return flow; + } + StateSyncRequestFlow::StateSyncRequestFlow( std::shared_ptr node_db, const primitives::BlockInfo &block_info, @@ -21,12 +37,7 @@ namespace kagome::network { : node_db_{std::move(node_db)}, block_info_{block_info}, block_{block}, - done_(isKnown(block.state_root)), - log_{log::createLogger("StateSync")} { - if (not done_) { - levels_.emplace_back(Level{.branch_hash = block.state_root}); - } - } + log_{log::createLogger("StateSync")} {} StateRequest StateSyncRequestFlow::nextRequest() const { BOOST_ASSERT(not complete()); @@ -56,7 +67,8 @@ namespace kagome::network { storage::trie::PolkadotCodec codec; BOOST_ASSERT(not complete()); BOOST_OUTCOME_TRY(auto nodes, storage::trie::compactDecode(res.proof)); - auto diff_count = nodes.size(), diff_size = res.proof.size(); + auto diff_count = nodes.size(); + auto diff_size = res.proof.size(); if (diff_count != 0) { stat_count_ += diff_count; stat_size_ += diff_size; @@ -76,7 +88,7 @@ namespace kagome::network { // when trie node is contained in other node value BOOST_OUTCOME_TRY(node, codec.decodeNode(raw)); } - level.push({ + OUTCOME_TRY(level.push({ .node = node, .branch = std::nullopt, .child = level.child, @@ -85,7 +97,7 @@ namespace kagome::network { .hash = it->first, .encoded = std::move(raw), }, - }); + })); nodes.erase(it); return outcome::success(); }; @@ -99,22 +111,30 @@ namespace kagome::network { auto pop_level = true; while (not level.stack.empty()) { auto child = level.value_child; - if (child and not isKnown(*child)) { + OUTCOME_TRY(known, isKnown(*child)); + if (child and not known) { auto &level = levels_.emplace_back(); level.branch_hash = child; pop_level = false; break; } - if (level.value_hash and not isKnown(*level.value_hash)) { - auto it = nodes.find(*level.value_hash); - if (it == nodes.end()) { - return outcome::success(); + if (level.value_hash) { + OUTCOME_TRY(known_value, isKnown(*level.value_hash)); + if (not known_value) { + auto it = nodes.find(*level.value_hash); + if (it == nodes.end()) { + return outcome::success(); + } + OUTCOME_TRY( + node_db_->values().put(it->first, std::move(it->second.first))); + known_.emplace(it->first); } - OUTCOME_TRY(node_db_->put(it->first, std::move(it->second.first))); - known_.emplace(it->first); } - for (level.branchInit(); not level.branch_end; level.branchNext()) { - if (not level.branch_hash or isKnown(*level.branch_hash)) { + OUTCOME_TRY(level.branchInit()); + while (not level.branch_end) { + OUTCOME_TRY(known, isKnown(*level.branch_hash)); + if (not level.branch_hash or known) { + OUTCOME_TRY(level.branchNext()); continue; } auto it = nodes.find(*level.branch_hash); @@ -126,11 +146,11 @@ namespace kagome::network { } if (level.branch_end) { auto &t = level.stack.back().t; - OUTCOME_TRY(node_db_->put(t.hash, std::move(t.encoded))); + OUTCOME_TRY(node_db_->nodes().put(t.hash, std::move(t.encoded))); known_.emplace(t.hash); - level.pop(); + OUTCOME_TRY(level.pop()); if (not level.stack.empty()) { - level.branchNext(); + OUTCOME_TRY(level.branchNext()); } } } @@ -142,16 +162,17 @@ namespace kagome::network { return outcome::success(); } - bool StateSyncRequestFlow::isKnown(const common::Hash256 &hash) { + outcome::result StateSyncRequestFlow::isKnown( + const common::Hash256 &hash) { if (hash == storage::trie::kEmptyRootHash) { return true; } if (known_.find(hash) != known_.end()) { return true; } - if (auto node_res = node_db_->contains(hash), - value_res = node_db_->contains(hash); - (node_res and node_res.value()) or (value_res and value_res.value())) { + OUTCOME_TRY(known_node, node_db_->nodes().contains(hash)); + OUTCOME_TRY(known_value, node_db_->values().contains(hash)); + if (known_node || known_value) { known_.emplace(hash); return true; } diff --git a/core/network/impl/state_sync_request_flow.hpp b/core/network/impl/state_sync_request_flow.hpp index 2c7cc0a2fc..b0b1a69948 100644 --- a/core/network/impl/state_sync_request_flow.hpp +++ b/core/network/impl/state_sync_request_flow.hpp @@ -33,7 +33,7 @@ namespace kagome::network { using Level = storage::trie::RawCursor; - StateSyncRequestFlow( + static outcome::result> create( std::shared_ptr node_db, const primitives::BlockInfo &block_info, const primitives::BlockHeader &block); @@ -55,7 +55,12 @@ namespace kagome::network { outcome::result onResponse(const StateResponse &res); private: - bool isKnown(const common::Hash256 &hash); + StateSyncRequestFlow( + std::shared_ptr node_db, + const primitives::BlockInfo &block_info, + const primitives::BlockHeader &block); + + outcome::result isKnown(const common::Hash256 &hash); std::shared_ptr node_db_; diff --git a/core/network/impl/synchronizer_impl.cpp b/core/network/impl/synchronizer_impl.cpp index a2f9292b37..5ab880162c 100644 --- a/core/network/impl/synchronizer_impl.cpp +++ b/core/network/impl/synchronizer_impl.cpp @@ -19,6 +19,7 @@ #include "consensus/grandpa/environment.hpp" #include "consensus/grandpa/has_authority_set_change.hpp" #include "consensus/timeline/timeline.hpp" +#include "network/impl/state_sync_request_flow.hpp" #include "network/peer_manager.hpp" #include "network/protocols/state_protocol.hpp" #include "network/protocols/sync_protocol.hpp" @@ -821,7 +822,13 @@ namespace kagome::network { return; } if (not state_sync_flow_ or state_sync_flow_->blockInfo() != block) { - state_sync_flow_.emplace(trie_node_db_, block, header); + auto flow_res = + StateSyncRequestFlow::create(trie_node_db_, block, header); + if (not flow_res) { + handler(flow_res.error()); + return; + } + state_sync_flow_.emplace(std::move(*flow_res.value())); } state_sync_.emplace(StateSync{ .peer = peer_id, diff --git a/core/parachain/approval/approval_distribution.hpp b/core/parachain/approval/approval_distribution.hpp index fab2b29bc1..608b9a195c 100644 --- a/core/parachain/approval/approval_distribution.hpp +++ b/core/parachain/approval/approval_distribution.hpp @@ -241,8 +241,9 @@ namespace kagome::parachain { CandidateEntry(const network::CandidateReceipt &receipt, SessionIndex session_index, size_t approvals_size) - : CandidateEntry( - HashedCandidateReceipt{receipt}, session_index, approvals_size) {} + : CandidateEntry(HashedCandidateReceipt{receipt}, + session_index, + approvals_size) {} std::optional> approval_entry( const network::RelayHash &relay_hash) { diff --git a/core/parachain/validator/statement_distribution/statement_distribution.cpp b/core/parachain/validator/statement_distribution/statement_distribution.cpp index c787e8842c..ba31755c2c 100644 --- a/core/parachain/validator/statement_distribution/statement_distribution.cpp +++ b/core/parachain/validator/statement_distribution/statement_distribution.cpp @@ -120,8 +120,10 @@ namespace kagome::parachain::statement_distribution { LazySPtr _slots_util, std::shared_ptr _babe_config_repo, primitives::events::PeerSubscriptionEnginePtr _peer_events_engine) - : implicit_view( - _prospective_parachains, _parachain_host, _block_tree, std::nullopt), + : implicit_view(_prospective_parachains, + _parachain_host, + _block_tree, + std::nullopt), per_session(RefCache::create()), signer_factory(std::move(sf)), peer_use_count( diff --git a/core/primitives/event_types.hpp b/core/primitives/event_types.hpp index 4c09737071..2c808a4b60 100644 --- a/core/primitives/event_types.hpp +++ b/core/primitives/event_types.hpp @@ -307,7 +307,7 @@ namespace kagome::primitives::events { struct ChainSub { ChainSub(ChainSubscriptionEnginePtr engine) : sub{std::make_shared( - std::move(engine))} {} + std::move(engine))} {} void onBlock(ChainEventType type, auto f) { subscribe(*sub, type, [f{std::move(f)}](const ChainEventParams &args) { diff --git a/core/runtime/common/module_repository_impl.cpp b/core/runtime/common/module_repository_impl.cpp index c4bbb4eb89..6d80480502 100644 --- a/core/runtime/common/module_repository_impl.cpp +++ b/core/runtime/common/module_repository_impl.cpp @@ -74,7 +74,7 @@ namespace kagome::runtime { KAGOME_PROFILE_START(module_retrieval) Item item; - OUTCOME_TRY(SAFE_UNIQUE(cache_)->outcome::result { + auto res = SAFE_UNIQUE(cache_)->outcome::result { if (auto r = cache_.get(state)) { item = r->get(); } else { @@ -93,7 +93,8 @@ namespace kagome::runtime { cache_.put(state, item); } return outcome::success(); - }); + }; + OUTCOME_TRY(res); return item; } } // namespace kagome::runtime diff --git a/core/runtime/common/trie_storage_provider_impl.cpp b/core/runtime/common/trie_storage_provider_impl.cpp index 7546f5324a..93f70d3169 100644 --- a/core/runtime/common/trie_storage_provider_impl.cpp +++ b/core/runtime/common/trie_storage_provider_impl.cpp @@ -158,7 +158,7 @@ namespace kagome::runtime { // TODO(turuslan): #2067, clone batch or implement delta_trie_root auto child_apply = [&](BufferView child, - storage::BufferStorage &map) -> outcome::result { + storage::trie::TrieBatch &map) -> outcome::result { for (auto &transaction : transaction_stack_) { auto it = transaction.child_batches.find(child); if (it == transaction.child_batches.end()) { diff --git a/core/storage/CMakeLists.txt b/core/storage/CMakeLists.txt index 2c69d8bb5b..eb6ef065e7 100644 --- a/core/storage/CMakeLists.txt +++ b/core/storage/CMakeLists.txt @@ -12,14 +12,12 @@ add_library(storage rocksdb/rocksdb_spaces.cpp database_error.cpp changes_trie/impl/storage_changes_tracker_impl.cpp - in_memory/in_memory_storage.cpp trie/child_prefix.cpp trie/compact_decode.cpp trie/compact_encode.cpp trie/impl/trie_batch_base.cpp trie/impl/ephemeral_trie_batch_impl.cpp trie/impl/trie_storage_impl.cpp - trie/impl/trie_storage_backend_batch.cpp trie/impl/trie_storage_backend_impl.cpp trie/impl/persistent_trie_batch_impl.cpp trie/impl/topper_trie_batch_impl.cpp @@ -31,6 +29,7 @@ add_library(storage trie/serialization/trie_serializer_impl.cpp trie/serialization/polkadot_codec.cpp trie_pruner/impl/trie_pruner_impl.cpp + migrations/migrations.cpp ) target_link_libraries(storage blob diff --git a/core/storage/buffer_map_types.hpp b/core/storage/buffer_map_types.hpp index 488739feb5..8c3904d1e5 100644 --- a/core/storage/buffer_map_types.hpp +++ b/core/storage/buffer_map_types.hpp @@ -13,9 +13,9 @@ #include "common/buffer.hpp" #include "common/buffer_or_view.hpp" -#include "storage/face/batch_writeable.hpp" #include "storage/face/generic_maps.hpp" #include "storage/face/write_batch.hpp" +#include "storage/spaces.hpp" namespace kagome::storage::face { template <> @@ -36,7 +36,10 @@ namespace kagome::storage { using BufferBatch = face::WriteBatch; + using BufferSpacedBatch = face::SpacedBatch; + using BufferStorage = face::GenericStorage; + using BufferBatchableStorage = face::BatchableStorage; using BufferStorageCursor = face::MapCursor; } // namespace kagome::storage diff --git a/core/storage/face/batch_writeable.hpp b/core/storage/face/batch_writeable.hpp index 6077980e53..c8546452a5 100644 --- a/core/storage/face/batch_writeable.hpp +++ b/core/storage/face/batch_writeable.hpp @@ -9,7 +9,6 @@ #include #include "storage/face/write_batch.hpp" -#include "storage/face/writeable.hpp" namespace kagome::storage::face { @@ -27,9 +26,7 @@ namespace kagome::storage::face { * @brief Creates new Write Batch - an object, which can be used to * efficiently write bulk data. */ - virtual std::unique_ptr> batch() { - throw std::logic_error{"BatchWriteable::batch not implemented"}; - } + virtual std::unique_ptr> batch() = 0; }; } // namespace kagome::storage::face diff --git a/core/storage/face/generic_maps.hpp b/core/storage/face/generic_maps.hpp index 8787d141b0..fdd1480881 100644 --- a/core/storage/face/generic_maps.hpp +++ b/core/storage/face/generic_maps.hpp @@ -12,16 +12,15 @@ #include "storage/face/writeable.hpp" namespace kagome::storage::face { + /** - * @brief An abstraction over a readable, writeable, iterable key-value map. + * @brief An abstraction over a readable, writeable, iterable, batchable + * key-value map. * @tparam K key type * @tparam V value type */ template - struct GenericStorage : Readable, - Iterable, - Writeable, - BatchWriteable { + struct GenericStorage : Readable, Iterable, Writeable { /** * Reports RAM state size * @return size in bytes @@ -31,4 +30,7 @@ namespace kagome::storage::face { } }; + template + struct BatchableStorage : GenericStorage, BatchWriteable {}; + } // namespace kagome::storage::face diff --git a/core/storage/face/write_batch.hpp b/core/storage/face/write_batch.hpp index 55b6bf49c6..102fe46a4b 100644 --- a/core/storage/face/write_batch.hpp +++ b/core/storage/face/write_batch.hpp @@ -29,4 +29,43 @@ namespace kagome::storage::face { virtual void clear() = 0; }; + /** + * @brief An abstraction over a spaced storage, which can be used for batch + * writes + * @tparam K key type + * @tparam V value type + */ + template + struct SpacedBatch { + virtual ~SpacedBatch() = default; + + /** + * @brief Store value by key + * @param key key + * @param value value + * @return result containing void if put successful, error otherwise + */ + virtual outcome::result put(Space space, + const View &key, + OwnedOrView &&value) = 0; + + /** + * @brief Remove value by key + * @param key K + * @return error code if error happened + */ + virtual outcome::result remove(Space space, const View &key) = 0; + + /** + * @brief Writes batch. + * @return error code in case of error. + */ + virtual outcome::result commit() = 0; + + /** + * @brief Clear batch. + */ + virtual void clear() = 0; + }; + } // namespace kagome::storage::face diff --git a/core/storage/in_memory/in_memory_batch.hpp b/core/storage/in_memory/in_memory_batch.hpp deleted file mode 100644 index f73e2c42c6..0000000000 --- a/core/storage/in_memory/in_memory_batch.hpp +++ /dev/null @@ -1,47 +0,0 @@ -/** - * Copyright Quadrivium LLC - * All Rights Reserved - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once - -#include "common/buffer.hpp" -#include "storage/in_memory/in_memory_storage.hpp" - -namespace kagome::storage { - using kagome::common::Buffer; - - class InMemoryBatch : public BufferBatch { - public: - explicit InMemoryBatch(InMemoryStorage &db) : db{db} {} - - outcome::result put(const BufferView &key, - BufferOrView &&value) override { - entries[key.toHex()] = std::move(value).intoBuffer(); - return outcome::success(); - } - - outcome::result remove(const BufferView &key) override { - entries.erase(key.toHex()); - return outcome::success(); - } - - outcome::result commit() override { - for (auto &entry : entries) { - OUTCOME_TRY(db.put(Buffer::fromHex(entry.first).value(), - BufferView{entry.second})); - } - return outcome::success(); - } - - void clear() override { - entries.clear(); - } - - private: - std::map entries; - // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) - InMemoryStorage &db; - }; -} // namespace kagome::storage diff --git a/core/storage/map_prefix/prefix.cpp b/core/storage/map_prefix/prefix.cpp index 8ad19f3280..ce9a045ac1 100644 --- a/core/storage/map_prefix/prefix.cpp +++ b/core/storage/map_prefix/prefix.cpp @@ -7,6 +7,7 @@ #include "storage/map_prefix/prefix.hpp" #include +#include "storage/buffer_map_types.hpp" namespace kagome::storage { inline std::optional afterPrefix(Buffer key) { @@ -99,7 +100,8 @@ namespace kagome::storage { batch->clear(); } - MapPrefix::MapPrefix(BufferView prefix, std::shared_ptr map) + MapPrefix::MapPrefix(BufferView prefix, + std::shared_ptr map) : prefix{prefix}, after_prefix{afterPrefix(this->prefix)}, map{std::move(map)} {} diff --git a/core/storage/map_prefix/prefix.hpp b/core/storage/map_prefix/prefix.hpp index 48d42b5b5e..cd055be9d4 100644 --- a/core/storage/map_prefix/prefix.hpp +++ b/core/storage/map_prefix/prefix.hpp @@ -7,13 +7,14 @@ #pragma once #include "storage/buffer_map_types.hpp" +#include "storage/face/batch_writeable.hpp" namespace kagome::storage { /** * Map wrapper to use keys under prefix. * Cursor removes key prefix and can seeks first/last. */ - struct MapPrefix : BufferStorage { + struct MapPrefix : BufferBatchableStorage { struct Cursor : BufferStorageCursor { Cursor(MapPrefix &map, std::unique_ptr cursor); @@ -45,7 +46,7 @@ namespace kagome::storage { std::unique_ptr batch; }; - MapPrefix(BufferView prefix, std::shared_ptr map); + MapPrefix(BufferView prefix, std::shared_ptr map); Buffer _key(BufferView key) const; outcome::result contains(const BufferView &key) const override; @@ -60,6 +61,6 @@ namespace kagome::storage { Buffer prefix; std::optional after_prefix; - std::shared_ptr map; + std::shared_ptr map; }; } // namespace kagome::storage diff --git a/core/storage/migrations/migrations.cpp b/core/storage/migrations/migrations.cpp new file mode 100644 index 0000000000..9b3a2e7184 --- /dev/null +++ b/core/storage/migrations/migrations.cpp @@ -0,0 +1,164 @@ +#include "storage/migrations/migrations.hpp" + +#include +#include + +#include +#include + +#include "blockchain/block_tree.hpp" +#include "blockchain/block_tree_error.hpp" +#include "common/blob.hpp" +#include "injector/application_injector.hpp" +#include "log/logger.hpp" +#include "primitives/common.hpp" +#include "runtime/runtime_upgrade_tracker.hpp" +#include "storage/database_error.hpp" +#include "storage/spaced_storage.hpp" +#include "storage/trie/trie_batches.hpp" +#include "storage/trie/trie_storage.hpp" +#include "storage/trie/trie_storage_backend.hpp" + +namespace kagome::storage::migrations { + + outcome::result migrateTree(SpacedStorage &storage, + trie::TrieBatch &trie_batch, + log::Logger &logger) { + auto batch = storage.createBatch(); + auto cursor = trie_batch.trieCursor(); + OUTCOME_TRY(cursor->next()); + + auto nodes = storage.getSpace(Space::kTrieNode); + auto values = storage.getSpace(Space::kTrieValue); + size_t migrated_values = 0; + size_t total_values = 0; + size_t small_values = 0; + while (cursor->isValid()) { + auto value_hash = cursor->valueHash(); + BOOST_ASSERT(value_hash.has_value()); // because cursor isValid + if (!value_hash->small) { + OUTCOME_TRY(present_in_values, values->contains(value_hash->hash)); + if (!present_in_values) { + OUTCOME_TRY(value, nodes->get(value_hash->hash)); + OUTCOME_TRY(batch->put( + Space::kTrieValue, value_hash->hash, std::move(value))); + OUTCOME_TRY(batch->remove(Space::kTrieNode, value_hash->hash)); + migrated_values++; + } + } else { + small_values++; + } + total_values++; + OUTCOME_TRY(cursor->next()); + } + SL_VERBOSE(logger, + "total values: {}, migrated values: {}, small values: {}", + total_values, + migrated_values, + small_values); + OUTCOME_TRY(batch->commit()); + return outcome::success(); + } + + outcome::result separateTrieValues( + const blockchain::BlockTree &block_tree, + const trie::TrieStorage &trie_storage, + SpacedStorage &storage, + runtime::RuntimeUpgradeTracker &upgrade_tracker) { + auto logger = log::createLogger("Migration", "storage"); + SL_INFO(logger, + "Begin trie storage migration to separate nodes and values"); + if (storage.getSpace(Space::kTrieValue)->cursor()->isValid()) { + SL_INFO(logger, + "Stop trie storage migration, trie values column is not empty " + "(migration is not required)."); + return outcome::success(); + } + + std::deque pending; + { + OUTCOME_TRY(children, + block_tree.getChildren(block_tree.getLastFinalized().hash)); + std::ranges::copy(children, std::back_inserter(pending)); + } + while (!pending.empty()) { + primitives::BlockHash current = pending.front(); + pending.pop_front(); + auto header = block_tree.getBlockHeader(current); + if (!header) { + if (header.error() != blockchain::BlockTreeError::HEADER_NOT_FOUND) { + return header.error(); + } + continue; + } + OUTCOME_TRY(children, block_tree.getChildren(current)); + std::ranges::copy(children, std::back_inserter(pending)); + SL_VERBOSE(logger, "Migrating block {}...", header.value().blockInfo()); + OUTCOME_TRY(batch, + trie_storage.getEphemeralBatchAt(header.value().state_root)); + OUTCOME_TRY(migrateTree(storage, *batch, logger)); + } + + { + // we also certainly need the block with the runtime code + OUTCOME_TRY(upgrade_state, + upgrade_tracker.getLastCodeUpdateState( + block_tree.getLastFinalized())); + OUTCOME_TRY(block, + upgrade_tracker.getLastCodeUpdateBlockInfo(upgrade_state)); + SL_VERBOSE(logger, "Migrating block {}...", block); + + OUTCOME_TRY(batch, trie_storage.getEphemeralBatchAt(upgrade_state)); + OUTCOME_TRY(migrateTree(storage, *batch, logger)); + } + auto header = block_tree.getBlockHeader(block_tree.getLastFinalized().hash); + OUTCOME_TRY(header); + { + SL_VERBOSE( + logger, "Migrating block {}...", block_tree.getLastFinalized()); + + OUTCOME_TRY(batch, + trie_storage.getEphemeralBatchAt(header.value().state_root)); + OUTCOME_TRY(migrateTree(storage, *batch, logger)); + } + SL_INFO(logger, + "Essential blocks have been migrated. In case that other finalized " + "blocks are not required, the migration may be stopped, because it " + "will take a long time. It can be restarted later, if needed."); + header = block_tree.getBlockHeader(header.value().parent_hash); + + for (; header.has_value(); + header = block_tree.getBlockHeader(header.value().parent_hash)) { + SL_VERBOSE(logger, "Migrating block {}...", header.value().blockInfo()); + auto trie_batch_res = + trie_storage.getEphemeralBatchAt(header.value().state_root); + if (!trie_batch_res) { + SL_VERBOSE(logger, + "State trie for block #{} is absent, assume we've reached " + "fast-synced blocks.", + header.value().number); + break; + } + + OUTCOME_TRY(migrateTree(storage, *trie_batch_res.value(), logger)); + } + if (header.has_error() + && header.error() != blockchain::BlockTreeError::HEADER_NOT_FOUND) { + return header.error(); + } + + SL_INFO(logger, "Trie storage migration ended successfully"); + return outcome::success(); + } + + outcome::result runMigrations(injector::KagomeNodeInjector &injector) { + auto block_tree = injector.injectBlockTree(); + auto trie_storage = injector.injectTrieStorage(); + auto storage = injector.injectStorage(); + auto upgrade_tracker = injector.injectRuntimeUpgradeTracker(); + + OUTCOME_TRY(separateTrieValues( + *block_tree, *trie_storage, *storage, *upgrade_tracker)); + return outcome::success(); + } +} // namespace kagome::storage::migrations diff --git a/core/storage/migrations/migrations.hpp b/core/storage/migrations/migrations.hpp new file mode 100644 index 0000000000..1be8146452 --- /dev/null +++ b/core/storage/migrations/migrations.hpp @@ -0,0 +1,34 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include "injector/application_injector.hpp" + +namespace kagome { + namespace blockchain { + class BlockTree; + } + namespace storage::trie { + class TrieStorage; + class TrieStorageBackend; + } // namespace storage::trie + namespace injector { + class KagomeNodeInjector; + } +} // namespace kagome + +namespace kagome::storage::migrations { + + outcome::result separateTrieValues( + const blockchain::BlockTree &block_tree, + const trie::TrieStorage &trie_storage, + trie::TrieStorageBackend &trie_backend); + + outcome::result runMigrations(injector::KagomeNodeInjector &injector); + +} // namespace kagome::storage::migrations diff --git a/core/storage/rocksdb/rocksdb.cpp b/core/storage/rocksdb/rocksdb.cpp index 82b37f8437..8e1623026c 100644 --- a/core/storage/rocksdb/rocksdb.cpp +++ b/core/storage/rocksdb/rocksdb.cpp @@ -5,11 +5,14 @@ */ #include "storage/rocksdb/rocksdb.hpp" +#include #include #include +#include #include "filesystem/common.hpp" +#include "storage/buffer_map_types.hpp" #include "storage/database_error.hpp" #include "storage/rocksdb/rocksdb_batch.hpp" #include "storage/rocksdb/rocksdb_cursor.hpp" @@ -37,7 +40,7 @@ namespace kagome::storage { bool prevent_destruction, const std::unordered_map &column_ttl, bool enable_migration) { - const auto no_db_presented = not fs::exists(path); + const auto no_db_exists = not fs::exists(path); OUTCOME_TRY(mkdirs(path)); auto log = log::createLogger("RocksDB", "storage"); @@ -76,7 +79,7 @@ namespace kagome::storage { const auto ttl_migrated_path = path.parent_path() / "ttl_migrated"; const auto ttl_migrated_exists = fs::exists(ttl_migrated_path); - if (no_db_presented or ttl_migrated_exists) { + if (no_db_exists or ttl_migrated_exists) { OUTCOME_TRY(openDatabaseWithTTL(options, path, column_family_descriptors, @@ -168,6 +171,12 @@ namespace kagome::storage { status.ToString()); return status_as_error(status); } + for (auto *handle : rocks_db->column_family_handles_) { + auto space = spaceByName(handle->GetName()); + BOOST_ASSERT(space.has_value()); + rocks_db->spaces_[*space] = std::make_shared( + rocks_db->weak_from_this(), *space, rocks_db->logger_); + } if (not fs::exists(ttl_migrated_path)) { std::ofstream file(ttl_migrated_path.native()); if (not file) { @@ -191,7 +200,7 @@ namespace kagome::storage { const filesystem::path &ttl_migrated_path, log::Logger &log) { rocksdb::DB *db_raw = nullptr; - std::vector column_family_handles; + std::vector column_family_handles; auto status = rocksdb::DB::Open(options, path.native(), column_family_descriptors, @@ -208,7 +217,7 @@ namespace kagome::storage { auto defer_db = std::make_unique(db, column_family_handles, log); - std::vector column_family_handles_with_ttl; + std::vector column_family_handles_with_ttl; const auto ttl_path = path.parent_path() / "db_ttl"; std::error_code ec; fs::create_directories(ttl_path, ec); @@ -280,68 +289,39 @@ namespace kagome::storage { ec); return DatabaseError::IO_ERROR; } - status = rocksdb::DBWithTTL::Open(options, - path.native(), - column_family_descriptors, - &rocks_db->column_family_handles_, - &rocks_db->db_, - ttls); - if (not status.ok()) { - SL_ERROR(log, - "Can't open database in {}: {}", - path.native(), - status.ToString()); - return status_as_error(status); - } - std::ofstream file(ttl_migrated_path.native()); - if (not file) { - SL_ERROR( - log, "Can't create file {} for database", ttl_migrated_path.native()); - return DatabaseError::IO_ERROR; - } - file.close(); - return outcome::success(); + + return openDatabaseWithTTL(options, + path, + column_family_descriptors, + ttls, + rocks_db, + ttl_migrated_path, + log); } - std::shared_ptr RocksDb::getSpace(Space space) { - if (spaces_.contains(space)) { - return spaces_[space]; - } - auto space_name = spaceName(space); - auto column = std::ranges::find_if( - column_family_handles_, - [&space_name](const ColumnFamilyHandlePtr &handle) { - return handle->GetName() == space_name; - }); - if (column_family_handles_.end() == column) { - throw DatabaseError::INVALID_ARGUMENT; - } - auto space_ptr = - std::make_shared(weak_from_this(), *column, logger_); - spaces_[space] = space_ptr; - return space_ptr; + std::shared_ptr RocksDb::getSpace(Space space) { + auto it = spaces_.find(space); + BOOST_ASSERT(it != spaces_.end()); + return it->second; } - void RocksDb::dropColumn(kagome::storage::Space space) { - auto space_name = spaceName(space); - auto column_it = std::ranges::find_if( - column_family_handles_, - [&space_name](const ColumnFamilyHandlePtr &handle) { - return handle->GetName() == space_name; - }); - if (column_family_handles_.end() == column_it) { - throw DatabaseError::INVALID_ARGUMENT; - } - auto &handle = *column_it; - auto e = [this](const rocksdb::Status &status) { + outcome::result RocksDb::dropColumn(kagome::storage::Space space) { + auto column = getCFHandle(space); + auto check_status = + [this](const rocksdb::Status &status) -> outcome::result { if (!status.ok()) { logger_->error("DB operation failed: {}", status.ToString()); - throw status_as_error(status); + return status_as_error(status); } + return outcome::success(); }; - e(db_->DropColumnFamily(handle)); - e(db_->DestroyColumnFamilyHandle(handle)); - e(db_->CreateColumnFamily({}, space_name, &handle)); + OUTCOME_TRY(check_status(db_->DropColumnFamily(column))); + OUTCOME_TRY(check_status(db_->DestroyColumnFamilyHandle(column))); + rocksdb::ColumnFamilyHandle *new_handle{}; + OUTCOME_TRY(check_status( + db_->CreateColumnFamily({}, spaceName(space), &new_handle))); + column_family_handles_[static_cast(space)] = new_handle; + return outcome::success(); } rocksdb::BlockBasedTableOptions RocksDb::tableOptionsConfiguration( @@ -349,13 +329,21 @@ namespace kagome::storage { rocksdb::BlockBasedTableOptions table_options; table_options.format_version = 5; table_options.block_cache = rocksdb::NewLRUCache( - static_cast(lru_cache_size_mib * 1024 * 1024)); - table_options.block_size = static_cast(block_size_kib * 1024); + static_cast(lru_cache_size_mib) * 1024 * 1024); + table_options.block_size = static_cast(block_size_kib) * 1024; table_options.cache_index_and_filter_blocks = true; table_options.filter_policy.reset(rocksdb::NewBloomFilterPolicy(10, false)); return table_options; } + rocksdb::ColumnFamilyHandle *RocksDb::getCFHandle(Space space) { + BOOST_ASSERT_MSG(static_cast(space) < column_family_handles_.size(), + "All spaces should have an associated column family"); + auto handle = column_family_handles_[static_cast(space)]; + BOOST_ASSERT(handle != nullptr); + return handle; + } + rocksdb::ColumnFamilyOptions RocksDb::configureColumn( uint32_t memory_budget) { rocksdb::ColumnFamilyOptions options; @@ -365,6 +353,11 @@ namespace kagome::storage { return options; } + std::unique_ptr RocksDb::createBatch() { + return std::make_unique(shared_from_this(), + getCFHandle(Space::kDefault)); + } + RocksDb::DatabaseGuard::DatabaseGuard( std::shared_ptr db, std::vector column_family_handles, @@ -413,16 +406,12 @@ namespace kagome::storage { } RocksDbSpace::RocksDbSpace(std::weak_ptr storage, - const RocksDb::ColumnFamilyHandlePtr &column, + Space space, log::Logger logger) : storage_{std::move(storage)}, - column_{column}, + space_{space}, logger_{std::move(logger)} {} - std::unique_ptr RocksDbSpace::batch() { - return std::make_unique(*this); - } - std::optional RocksDbSpace::byteSizeHint() const { auto rocks = storage_.lock(); if (!rocks) { @@ -446,20 +435,31 @@ namespace kagome::storage { return usage_bytes; } + std::unique_ptr> RocksDbSpace::batch() { + auto rocks = storage_.lock(); + if (!rocks) { + throw DatabaseError::STORAGE_GONE; + } + auto batch = + std::make_unique(rocks, rocks->getCFHandle(space_)); + return batch; + } + std::unique_ptr RocksDbSpace::cursor() { auto rocks = storage_.lock(); if (!rocks) { throw DatabaseError::STORAGE_GONE; } auto it = std::unique_ptr( - rocks->db_->NewIterator(rocks->ro_, column_)); + rocks->db_->NewIterator(rocks->ro_, rocks->getCFHandle(space_))); return std::make_unique(std::move(it)); } outcome::result RocksDbSpace::contains(const BufferView &key) const { OUTCOME_TRY(rocks, use()); std::string value; - auto status = rocks->db_->Get(rocks->ro_, column_, make_slice(key), &value); + auto status = rocks->db_->Get( + rocks->ro_, rocks->getCFHandle(space_), make_slice(key), &value); if (status.ok()) { return true; } @@ -474,7 +474,8 @@ namespace kagome::storage { outcome::result RocksDbSpace::get(const BufferView &key) const { OUTCOME_TRY(rocks, use()); std::string value; - auto status = rocks->db_->Get(rocks->ro_, column_, make_slice(key), &value); + auto status = rocks->db_->Get( + rocks->ro_, rocks->getCFHandle(space_), make_slice(key), &value); if (status.ok()) { // cannot move string content to a buffer return Buffer( @@ -488,7 +489,8 @@ namespace kagome::storage { const BufferView &key) const { OUTCOME_TRY(rocks, use()); std::string value; - auto status = rocks->db_->Get(rocks->ro_, column_, make_slice(key), &value); + auto status = rocks->db_->Get( + rocks->ro_, rocks->getCFHandle(space_), make_slice(key), &value); if (status.ok()) { auto buf = Buffer( reinterpret_cast(value.data()), // NOLINT @@ -506,8 +508,10 @@ namespace kagome::storage { outcome::result RocksDbSpace::put(const BufferView &key, BufferOrView &&value) { OUTCOME_TRY(rocks, use()); - auto status = rocks->db_->Put( - rocks->wo_, column_, make_slice(key), make_slice(std::move(value))); + auto status = rocks->db_->Put(rocks->wo_, + rocks->getCFHandle(space_), + make_slice(key), + make_slice(std::move(value))); if (status.ok()) { return outcome::success(); } @@ -517,7 +521,8 @@ namespace kagome::storage { outcome::result RocksDbSpace::remove(const BufferView &key) { OUTCOME_TRY(rocks, use()); - auto status = rocks->db_->Delete(rocks->wo_, column_, make_slice(key)); + auto status = rocks->db_->Delete( + rocks->wo_, rocks->getCFHandle(space_), make_slice(key)); if (status.ok()) { return outcome::success(); } @@ -532,15 +537,15 @@ namespace kagome::storage { } if (rocks->db_) { std::unique_ptr begin( - rocks->db_->NewIterator(rocks->ro_, column_)); + rocks->db_->NewIterator(rocks->ro_, rocks->getCFHandle(space_))); first.empty() ? begin->SeekToFirst() : begin->Seek(make_slice(first)); auto bk = begin->key(); std::unique_ptr end( - rocks->db_->NewIterator(rocks->ro_, column_)); + rocks->db_->NewIterator(rocks->ro_, rocks->getCFHandle(space_))); last.empty() ? end->SeekToLast() : end->Seek(make_slice(last)); auto ek = end->key(); rocksdb::CompactRangeOptions options; - rocks->db_->CompactRange(options, column_, &bk, &ek); + rocks->db_->CompactRange(options, rocks->getCFHandle(space_), &bk, &ek); } } diff --git a/core/storage/rocksdb/rocksdb.hpp b/core/storage/rocksdb/rocksdb.hpp index f3d6aec89d..79969870fb 100644 --- a/core/storage/rocksdb/rocksdb.hpp +++ b/core/storage/rocksdb/rocksdb.hpp @@ -6,6 +6,7 @@ #pragma once +#include "common/buffer.hpp" #include "storage/buffer_map_types.hpp" #include @@ -15,15 +16,14 @@ #include "filesystem/common.hpp" #include "log/logger.hpp" +#include "storage/face/batch_writeable.hpp" +#include "storage/rocksdb/rocksdb_spaces.hpp" #include "storage/spaced_storage.hpp" namespace kagome::storage { class RocksDb : public SpacedStorage, public std::enable_shared_from_this { - private: - using ColumnFamilyHandlePtr = rocksdb::ColumnFamilyHandle *; - public: ~RocksDb() override; @@ -53,14 +53,16 @@ namespace kagome::storage { const std::unordered_map &column_ttl = {}, bool enable_migration = true); - std::shared_ptr getSpace(Space space) override; + std::shared_ptr getSpace(Space space) override; + + std::unique_ptr createBatch() override; /** * Implementation specific way to erase the whole space data. * Not exposed at SpacedStorage level as only used in pruner. * @param space - storage space identifier to clear */ - void dropColumn(Space space); + outcome::result dropColumn(Space space); /** * Prepare configuration structure @@ -97,6 +99,9 @@ namespace kagome::storage { }; RocksDb(); + + rocksdb::ColumnFamilyHandle *getCFHandle(Space space); + static rocksdb::ColumnFamilyOptions configureColumn(uint32_t memory_budget); static outcome::result createDirectory( const std::filesystem::path &absolute_path, log::Logger &log); @@ -130,25 +135,26 @@ namespace kagome::storage { log::Logger &log); rocksdb::DBWithTTL *db_{}; - std::vector column_family_handles_; - boost::container::flat_map> spaces_; + std::vector column_family_handles_; + boost::container::flat_map> + spaces_; rocksdb::ReadOptions ro_; rocksdb::WriteOptions wo_; log::Logger logger_; }; - class RocksDbSpace : public BufferStorage { + class RocksDbSpace : public BufferBatchableStorage { public: ~RocksDbSpace() override = default; RocksDbSpace(std::weak_ptr storage, - const RocksDb::ColumnFamilyHandlePtr &column, + Space space, log::Logger logger); - std::unique_ptr batch() override; - std::optional byteSizeHint() const override; + std::unique_ptr> batch() override; + std::unique_ptr cursor() override; outcome::result contains(const BufferView &key) const override; @@ -166,14 +172,14 @@ namespace kagome::storage { void compact(const Buffer &first, const Buffer &last); friend class RocksDbBatch; + friend class RocksDb; private: // gather storage instance from weak ptr outcome::result> use() const; std::weak_ptr storage_; - // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) - const RocksDb::ColumnFamilyHandlePtr &column_; + Space space_; log::Logger logger_; }; } // namespace kagome::storage diff --git a/core/storage/rocksdb/rocksdb_batch.cpp b/core/storage/rocksdb/rocksdb_batch.cpp index 4910dfad84..505fbab55b 100644 --- a/core/storage/rocksdb/rocksdb_batch.cpp +++ b/core/storage/rocksdb/rocksdb_batch.cpp @@ -11,30 +11,39 @@ namespace kagome::storage { - RocksDbBatch::RocksDbBatch(RocksDbSpace &db) : db_(db) {} + RocksDbBatch::RocksDbBatch(std::shared_ptr db, + rocksdb::ColumnFamilyHandle *default_cf) + : db_(std::move(db)), default_cf_(default_cf) { + BOOST_ASSERT(db_ != nullptr); + BOOST_ASSERT(default_cf_ != nullptr); + } outcome::result RocksDbBatch::put(const BufferView &key, BufferOrView &&value) { - batch_.Put(db_.column_, make_slice(key), make_slice(std::move(value))); - return outcome::success(); + return status_as_result( + batch_.Put(default_cf_, make_slice(key), make_slice(std::move(value)))); + } + + outcome::result RocksDbBatch::put(Space space, + const BufferView &key, + BufferOrView &&value) { + auto handle = db_->getCFHandle(space); + return status_as_result( + batch_.Put(handle, make_slice(key), make_slice(std::move(value)))); } outcome::result RocksDbBatch::remove(const BufferView &key) { - batch_.Delete(db_.column_, make_slice(key)); - return outcome::success(); + return status_as_result(batch_.Delete(default_cf_, make_slice(key))); + } + + outcome::result RocksDbBatch::remove(Space space, + const BufferView &key) { + auto handle = db_->getCFHandle(space); + return status_as_result(batch_.Delete(handle, make_slice(key))); } outcome::result RocksDbBatch::commit() { - auto rocks = db_.storage_.lock(); - if (!rocks) { - return DatabaseError::STORAGE_GONE; - } - auto status = rocks->db_->Write(rocks->wo_, &batch_); - if (status.ok()) { - return outcome::success(); - } - - return status_as_error(status); + return status_as_result(db_->db_->Write(db_->wo_, &batch_)); } void RocksDbBatch::clear() { diff --git a/core/storage/rocksdb/rocksdb_batch.hpp b/core/storage/rocksdb/rocksdb_batch.hpp index e19476a5a6..e7039a1b1f 100644 --- a/core/storage/rocksdb/rocksdb_batch.hpp +++ b/core/storage/rocksdb/rocksdb_batch.hpp @@ -6,16 +6,22 @@ #pragma once +#include #include + +#include "common/buffer.hpp" +#include "storage/buffer_map_types.hpp" +#include "storage/face/write_batch.hpp" #include "storage/rocksdb/rocksdb.hpp" namespace kagome::storage { - class RocksDbBatch : public BufferBatch { + class RocksDbBatch : public BufferSpacedBatch, public BufferBatch { public: ~RocksDbBatch() override = default; - explicit RocksDbBatch(RocksDbSpace &db); + explicit RocksDbBatch(std::shared_ptr db, + rocksdb::ColumnFamilyHandle *default_cf); outcome::result commit() override; @@ -24,11 +30,16 @@ namespace kagome::storage { outcome::result put(const BufferView &key, BufferOrView &&value) override; + outcome::result put(Space space, + const BufferView &key, + BufferOrView &&value) override; + outcome::result remove(const BufferView &key) override; + outcome::result remove(Space space, const BufferView &key) override; private: - // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) - RocksDbSpace &db_; + std::shared_ptr db_; rocksdb::WriteBatch batch_; + rocksdb::ColumnFamilyHandle *default_cf_; }; } // namespace kagome::storage diff --git a/core/storage/rocksdb/rocksdb_spaces.cpp b/core/storage/rocksdb/rocksdb_spaces.cpp index c1f4f920c9..7eabdfc841 100644 --- a/core/storage/rocksdb/rocksdb_spaces.cpp +++ b/core/storage/rocksdb/rocksdb_spaces.cpp @@ -10,31 +10,43 @@ #include #include +#include namespace kagome::storage { + static constexpr std::array kSpaceNames{"lookup_key", + "header", + "block_body", + "justification", + "trie_node", + "trie_value", + "dispute_data", + "beefy_justification", + "avaliability_storage"}; std::string spaceName(Space space) { - static constexpr std::array kNames{ - "lookup_key", - "header", - "block_body", - "justification", - "trie_node", - "trie_value", - "dispute_data", - "beefy_justification", - "avaliability_storage" - }; - static_assert(kNames.size() == Space::kTotal - 1); + static_assert(kSpaceNames.size() == Space::kTotal - 1); static const std::vector names = []() { std::vector names; names.push_back(rocksdb::kDefaultColumnFamilyName); - names.insert(names.end(), kNames.begin(), kNames.end()); + names.insert(names.end(), kSpaceNames.begin(), kSpaceNames.end()); return names; }(); BOOST_ASSERT(space < Space::kTotal); return names.at(space); } + std::optional spaceByName(std::string_view space) { + if (space == rocksdb::kDefaultColumnFamilyName) { + return Space::kDefault; + } + auto it = std::ranges::find(kSpaceNames, space); + if (it == kSpaceNames.end()) { + return std::nullopt; + } + auto idx = it - kSpaceNames.begin() + 1; + BOOST_ASSERT(idx < Space::kTotal); + return static_cast(idx); + } + } // namespace kagome::storage diff --git a/core/storage/rocksdb/rocksdb_spaces.hpp b/core/storage/rocksdb/rocksdb_spaces.hpp index 41d8bc4039..2ef87c36ca 100644 --- a/core/storage/rocksdb/rocksdb_spaces.hpp +++ b/core/storage/rocksdb/rocksdb_spaces.hpp @@ -8,15 +8,19 @@ #include "storage/spaces.hpp" +#include #include namespace kagome::storage { /** - * Map space item to its string name for Rocks DB needs - * @param space - space identifier - * @return string representation of space name + * Get a space name as a string */ std::string spaceName(Space space); + /** + * Get the space by its name + */ + std::optional spaceByName(std::string_view space); + } // namespace kagome::storage diff --git a/core/storage/rocksdb/rocksdb_util.hpp b/core/storage/rocksdb/rocksdb_util.hpp index 162fbc9120..249f3fb456 100644 --- a/core/storage/rocksdb/rocksdb_util.hpp +++ b/core/storage/rocksdb/rocksdb_util.hpp @@ -7,10 +7,14 @@ #pragma once #include +#include + #include "common/buffer.hpp" +#include "log/logger.hpp" #include "storage/database_error.hpp" namespace kagome::storage { + inline DatabaseError status_as_error(const rocksdb::Status &s) { if (s.IsNotFound()) { return DatabaseError::NOT_FOUND; @@ -37,6 +41,13 @@ namespace kagome::storage { return DatabaseError::UNKNOWN; } + inline outcome::result status_as_result(const rocksdb::Status &s) { + if (s.ok()) { + return outcome::success(); + } + return status_as_error(s); + } + inline rocksdb::Slice make_slice(const common::BufferView &buf) { // NOLINTNEXTLINE(cppcoreguidelines-pro-type-reinterpret-cast) const auto *ptr = reinterpret_cast(buf.data()); diff --git a/core/storage/spaced_storage.hpp b/core/storage/spaced_storage.hpp index 8d8445d4fc..cff8e1be94 100644 --- a/core/storage/spaced_storage.hpp +++ b/core/storage/spaced_storage.hpp @@ -24,7 +24,13 @@ namespace kagome::storage { * @param space - identifier of required space * @return a pointer buffer storage for a space */ - virtual std::shared_ptr getSpace(Space space) = 0; + virtual std::shared_ptr getSpace(Space space) = 0; + + /** + * Retrieve a batch to write into the storage atomically + * @return a new batch + */ + virtual std::unique_ptr createBatch() = 0; }; } // namespace kagome::storage diff --git a/core/storage/trie/child_prefix.hpp b/core/storage/trie/child_prefix.hpp index 49319bdc1f..6ef8b1a97e 100644 --- a/core/storage/trie/child_prefix.hpp +++ b/core/storage/trie/child_prefix.hpp @@ -19,7 +19,7 @@ namespace kagome::storage::trie { void match(uint8_t nibble); void match(common::BufferView nibbles); - operator bool() const; + explicit operator bool() const; bool done() const; diff --git a/core/storage/trie/compact_decode.cpp b/core/storage/trie/compact_decode.cpp index 99d79fe9b6..631d785787 100644 --- a/core/storage/trie/compact_decode.cpp +++ b/core/storage/trie/compact_decode.cpp @@ -15,6 +15,8 @@ OUTCOME_CPP_DEFINE_CATEGORY(kagome::storage::trie, CompactDecodeError, e) { switch (e) { case E::INCOMPLETE_PROOF: return "incomplete proof"; + case E::NULL_BRANCH: + return "Unexpected null branch"; } abort(); } @@ -48,21 +50,23 @@ namespace kagome::storage::trie { db.emplace(hash, std::make_pair(std::move(value), nullptr)); node->setValue({std::nullopt, hash}); } - cursor.push({ + OUTCOME_TRY(cursor.push({ .node = node, .branch = 0, .child = false, .t = {}, - }); + })); return outcome::success(); }; OUTCOME_TRY(push()); while (not cursor.stack.empty()) { - for (cursor.branchInit(); not cursor.branch_end; cursor.branchNext()) { + OUTCOME_TRY(cursor.branchInit()); + while (not cursor.branch_end) { if (not cursor.branch_merkle) { - throw std::logic_error{"compactDecode branch_merkle=null"}; + return CompactDecodeError::NULL_BRANCH; } if (not cursor.branch_merkle->empty()) { + OUTCOME_TRY(cursor.branchNext()); continue; } OUTCOME_TRY(push()); @@ -73,10 +77,10 @@ namespace kagome::storage::trie { OUTCOME_TRY(raw, codec.encodeNode(*node, StateVersion::V0)); auto hash = codec.hash256(raw); db[hash] = {std::move(raw), std::move(node)}; - cursor.pop(); + OUTCOME_TRY(cursor.pop()); if (not cursor.stack.empty()) { *cursor.branch_merkle = hash; - cursor.branchNext(); + OUTCOME_TRY(cursor.branchNext()); } } } diff --git a/core/storage/trie/compact_decode.hpp b/core/storage/trie/compact_decode.hpp index f1b3889f35..8aacc222f7 100644 --- a/core/storage/trie/compact_decode.hpp +++ b/core/storage/trie/compact_decode.hpp @@ -16,6 +16,7 @@ namespace kagome::storage::trie { enum class CompactDecodeError : uint8_t { INCOMPLETE_PROOF = 1, + NULL_BRANCH, }; using CompactDecoded = diff --git a/core/storage/trie/compact_encode.cpp b/core/storage/trie/compact_encode.cpp index c0e8bbe2c4..ee7c18c9ca 100644 --- a/core/storage/trie/compact_encode.cpp +++ b/core/storage/trie/compact_encode.cpp @@ -6,11 +6,25 @@ #include "storage/trie/compact_encode.hpp" +#include #include #include "storage/trie/raw_cursor.hpp" #include "storage/trie/serialization/polkadot_codec.hpp" +OUTCOME_CPP_DEFINE_CATEGORY(kagome::storage::trie, RawCursorError, e) { + using E = kagome::storage::trie::RawCursorError; + switch (e) { + case E::EmptyStack: + return "Unexpected empty stack"; + case E::ChildBranchNotFound: + return "Expected child branch is not found"; + case E::StackBackIsNotBranch: + return "No branch at the end of the stack"; + } + return "Unknown RawCursorError"; +} + namespace kagome::storage::trie { outcome::result compactEncode(const OnRead &db, const common::Hash256 &root) { @@ -20,7 +34,7 @@ namespace kagome::storage::trie { level.child = {}; std::unordered_set seen, value_seen; std::vector> proofs(2); - auto push = [&](const common::Hash256 &hash) { + auto push = [&](const common::Hash256 &hash) -> outcome::result { auto it = db.db.find(hash); if (it == db.db.end()) { return false; @@ -42,11 +56,11 @@ namespace kagome::storage::trie { } auto &level = levels.back(); auto &proof = proofs[levels.size() - 1]; - level.push({ + OUTCOME_TRY(level.push({ .node = std::move(node), .child = level.child, .t = proof.size(), - }); + })); proof.emplace_back(); if (compact) { proof.back().putUint8(kEscapeCompactHeader); @@ -54,7 +68,7 @@ namespace kagome::storage::trie { } return true; }; - push(root); + OUTCOME_TRY(push(root)); while (not levels.empty()) { auto &level = levels.back(); auto pop_level = true; @@ -62,24 +76,27 @@ namespace kagome::storage::trie { auto child = level.value_child; if (child and seen.emplace(*child).second) { levels.emplace_back(); - push(*child); + OUTCOME_TRY(push(*child)); pop_level = false; break; } - for (level.branchInit(); not level.branch_end; level.branchNext()) { + OUTCOME_TRY(level.branchInit()); + while (not level.branch_end) { + OUTCOME_TRY(push_success, push(*level.branch_hash)); if (level.branch_hash and seen.emplace(*level.branch_hash).second - and push(*level.branch_hash)) { + and push_success) { break; } + OUTCOME_TRY(level.branchNext()); } if (level.branch_end) { auto &item = level.stack.back(); auto &proof = proofs[levels.size() - 1][item.t]; proof.put(codec.encodeNode(*item.node, StateVersion::V0).value()); - level.pop(); + OUTCOME_TRY(level.pop()); if (not level.stack.empty()) { *level.branch_merkle = MerkleValue::create({}).value(); - level.branchNext(); + OUTCOME_TRY(level.branchNext()); } } } diff --git a/core/storage/trie/impl/topper_trie_batch_impl.cpp b/core/storage/trie/impl/topper_trie_batch_impl.cpp index 48813eec8d..260fc2472f 100644 --- a/core/storage/trie/impl/topper_trie_batch_impl.cpp +++ b/core/storage/trie/impl/topper_trie_batch_impl.cpp @@ -9,8 +9,11 @@ #include #include "common/buffer.hpp" +#include "crypto/blake2/blake2b.h" +#include "storage/face/generic_maps.hpp" #include "storage/trie/polkadot_trie/polkadot_trie_cursor.hpp" #include "storage/trie/polkadot_trie/trie_error.hpp" +#include "storage/trie/polkadot_trie/trie_node.hpp" OUTCOME_CPP_DEFINE_CATEGORY(kagome::storage::trie, TopperTrieBatchImpl::Error, @@ -116,7 +119,7 @@ namespace kagome::storage::trie { } outcome::result TopperTrieBatchImpl::apply( - storage::BufferStorage &map) { + face::Writeable &map) { for (auto &[k, v] : cache_) { if (v) { OUTCOME_TRY(map.put(k, BufferView{*v})); @@ -161,7 +164,7 @@ namespace kagome::storage::trie { } bool TopperTrieCursor::isValid() const { - return choice_; + return static_cast(choice_); } outcome::result TopperTrieCursor::next() { @@ -183,6 +186,20 @@ namespace kagome::storage::trie { : parent_cursor_->value(); } + std::optional TopperTrieCursor::valueHash() + const { + auto value_opt = value(); + if (!value_opt) { + return std::nullopt; + } + if (value_opt->size() >= Hash256::size() + 1) { + return ValueHash{.hash = crypto::blake2b<32>(*value_opt), .small = false}; + } + Hash256 value_as_hash{}; + std::copy(value_opt->begin(), value_opt->end(), value_as_hash.begin()); + return ValueHash{.hash = value_as_hash, .small = true}; + } + outcome::result TopperTrieCursor::seekLowerBound( const BufferView &key) { OUTCOME_TRY(parent_cursor_->seekLowerBound(key)); diff --git a/core/storage/trie/impl/topper_trie_batch_impl.hpp b/core/storage/trie/impl/topper_trie_batch_impl.hpp index 97bd487ea4..1590e6a610 100644 --- a/core/storage/trie/impl/topper_trie_batch_impl.hpp +++ b/core/storage/trie/impl/topper_trie_batch_impl.hpp @@ -6,6 +6,11 @@ #pragma once +#include "common/blob.hpp" +#include "common/buffer.hpp" +#include "storage/buffer_map_types.hpp" +#include "storage/face/writeable.hpp" +#include "storage/trie/polkadot_trie/trie_node.hpp" #include "storage/trie/trie_batches.hpp" #include @@ -48,7 +53,7 @@ namespace kagome::storage::trie { outcome::result>> createChildBatch( common::BufferView path) override; - outcome::result apply(storage::BufferStorage &map); + outcome::result apply(face::Writeable &map); private: std::map> cache_; @@ -75,6 +80,7 @@ namespace kagome::storage::trie { outcome::result prev() override; std::optional key() const override; std::optional value() const override; + std::optional valueHash() const override; outcome::result seekLowerBound(const BufferView &key) override; outcome::result seekUpperBound(const BufferView &key) override; @@ -88,7 +94,7 @@ namespace kagome::storage::trie { struct Choice { Choice(bool parent, bool overlay) : parent{parent}, overlay{overlay} {} - operator bool() const { + explicit operator bool() const { return parent || overlay; } diff --git a/core/storage/trie/impl/trie_storage_backend_batch.cpp b/core/storage/trie/impl/trie_storage_backend_batch.cpp deleted file mode 100644 index 17e1393693..0000000000 --- a/core/storage/trie/impl/trie_storage_backend_batch.cpp +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright Quadrivium LLC - * All Rights Reserved - * SPDX-License-Identifier: Apache-2.0 - */ - -#include "storage/trie/impl/trie_storage_backend_batch.hpp" - -namespace kagome::storage::trie { - - TrieStorageBackendBatch::TrieStorageBackendBatch( - std::unique_ptr storage_batch) - : storage_batch_{std::move(storage_batch)} { - BOOST_ASSERT(storage_batch_ != nullptr); - } - - outcome::result TrieStorageBackendBatch::commit() { - return storage_batch_->commit(); - } - - void TrieStorageBackendBatch::clear() { - storage_batch_->clear(); - } - - outcome::result TrieStorageBackendBatch::put( - const common::BufferView &key, BufferOrView &&value) { - return storage_batch_->put(key, std::move(value)); - } - - outcome::result TrieStorageBackendBatch::remove( - const common::BufferView &key) { - return storage_batch_->remove(key); - } - -} // namespace kagome::storage::trie diff --git a/core/storage/trie/impl/trie_storage_backend_batch.hpp b/core/storage/trie/impl/trie_storage_backend_batch.hpp deleted file mode 100644 index 4785d530d1..0000000000 --- a/core/storage/trie/impl/trie_storage_backend_batch.hpp +++ /dev/null @@ -1,34 +0,0 @@ -/** - * Copyright Quadrivium LLC - * All Rights Reserved - * SPDX-License-Identifier: Apache-2.0 - */ - -#pragma once - -#include "storage/buffer_map_types.hpp" - -namespace kagome::storage::trie { - - /** - * Batch implementation for TrieStorageBackend - * @see TrieStorageBackend - */ - class TrieStorageBackendBatch : public BufferBatch { - public: - TrieStorageBackendBatch(std::unique_ptr storage_batch); - ~TrieStorageBackendBatch() override = default; - - outcome::result commit() override; - - outcome::result put(const common::BufferView &key, - BufferOrView &&value) override; - - outcome::result remove(const common::BufferView &key) override; - void clear() override; - - private: - std::unique_ptr storage_batch_; - }; - -} // namespace kagome::storage::trie diff --git a/core/storage/trie/impl/trie_storage_backend_impl.cpp b/core/storage/trie/impl/trie_storage_backend_impl.cpp index c8b60e5e73..cd8b26aae9 100644 --- a/core/storage/trie/impl/trie_storage_backend_impl.cpp +++ b/core/storage/trie/impl/trie_storage_backend_impl.cpp @@ -9,46 +9,5 @@ #include #include "storage/spaces.hpp" -#include "storage/trie/impl/trie_storage_backend_batch.hpp" -namespace kagome::storage::trie { - - TrieStorageBackendImpl::TrieStorageBackendImpl( - std::shared_ptr storage) - : storage_{storage->getSpace(Space::kTrieNode)} { - BOOST_ASSERT(storage_ != nullptr); - } - - std::unique_ptr - TrieStorageBackendImpl::cursor() { - return storage_->cursor(); - } - - std::unique_ptr TrieStorageBackendImpl::batch() { - return std::make_unique(storage_->batch()); - } - - outcome::result TrieStorageBackendImpl::get( - const BufferView &key) const { - return storage_->get(key); - } - - outcome::result> TrieStorageBackendImpl::tryGet( - const BufferView &key) const { - return storage_->tryGet(key); - } - - outcome::result TrieStorageBackendImpl::contains( - const BufferView &key) const { - return storage_->contains(key); - } - - outcome::result TrieStorageBackendImpl::put(const BufferView &key, - BufferOrView &&value) { - return storage_->put(key, std::move(value)); - } - - outcome::result TrieStorageBackendImpl::remove(const BufferView &key) { - return storage_->remove(key); - } -} // namespace kagome::storage::trie +namespace kagome::storage::trie {} // namespace kagome::storage::trie diff --git a/core/storage/trie/impl/trie_storage_backend_impl.hpp b/core/storage/trie/impl/trie_storage_backend_impl.hpp index 95786ad952..a015de06cb 100644 --- a/core/storage/trie/impl/trie_storage_backend_impl.hpp +++ b/core/storage/trie/impl/trie_storage_backend_impl.hpp @@ -15,24 +15,25 @@ namespace kagome::storage::trie { class TrieStorageBackendImpl : public TrieStorageBackend { public: - TrieStorageBackendImpl(std::shared_ptr storage); + TrieStorageBackendImpl(std::shared_ptr db) + : db_{std::move(db)} { + BOOST_ASSERT(db_ != nullptr); + } - ~TrieStorageBackendImpl() override = default; + BufferStorage &nodes() override { + return *db_->getSpace(Space::kTrieNode); + } - std::unique_ptr cursor() override; - std::unique_ptr batch() override; + BufferStorage &values() override { + return *db_->getSpace(Space::kTrieValue); + } - outcome::result get(const BufferView &key) const override; - outcome::result> tryGet( - const BufferView &key) const override; - outcome::result contains(const BufferView &key) const override; - - outcome::result put(const BufferView &key, - BufferOrView &&value) override; - outcome::result remove(const common::BufferView &key) override; + std::unique_ptr batch() override { + return db_->createBatch(); + } private: - std::shared_ptr storage_; + std::shared_ptr db_; }; } // namespace kagome::storage::trie diff --git a/core/storage/trie/polkadot_trie/polkadot_trie_cursor.hpp b/core/storage/trie/polkadot_trie/polkadot_trie_cursor.hpp index 6f7674caf8..173450a830 100644 --- a/core/storage/trie/polkadot_trie/polkadot_trie_cursor.hpp +++ b/core/storage/trie/polkadot_trie/polkadot_trie_cursor.hpp @@ -28,6 +28,14 @@ namespace kagome::storage::trie { */ virtual outcome::result seekUpperBound( const common::BufferView &key) = 0; + + // small values (less than hash size) are not hashed and stored as-is inside + // their node + struct ValueHash { + Hash256 hash; + bool small = false; + }; + [[nodiscard]] virtual std::optional valueHash() const = 0; }; } // namespace kagome::storage::trie diff --git a/core/storage/trie/polkadot_trie/polkadot_trie_cursor_impl.cpp b/core/storage/trie/polkadot_trie/polkadot_trie_cursor_impl.cpp index 76fc287a28..8aa3ae3675 100644 --- a/core/storage/trie/polkadot_trie/polkadot_trie_cursor_impl.cpp +++ b/core/storage/trie/polkadot_trie/polkadot_trie_cursor_impl.cpp @@ -8,6 +8,8 @@ #include +#include "common/blob.hpp" +#include "crypto/blake2/blake2b.h" #include "macro/unreachable.hpp" #include "storage/trie/polkadot_trie/polkadot_trie.hpp" #include "storage/trie/polkadot_trie/trie_error.hpp" @@ -351,6 +353,30 @@ namespace kagome::storage::trie { return std::nullopt; } + std::optional + PolkadotTrieCursorImpl::valueHash() const { + if (const auto *search_state = std::get_if(&state_); + search_state != nullptr) { + const auto &value_opt = search_state->getCurrent().getValue(); + + if (value_opt.hash) { + return ValueHash{.hash = value_opt.hash.value(), .small = false}; + } + if (value_opt.value) { + if (value_opt.value->size() >= Hash256::size() + 1) { + return ValueHash{.hash = crypto::blake2b<32>(value_opt.value.value()), + .small = false}; + } + Hash256 value_as_hash{}; + std::copy(value_opt.value.value().begin(), + value_opt.value.value().end(), + value_as_hash.begin()); + return ValueHash{.hash = value_as_hash, .small = true}; + } + } + return std::nullopt; + } + auto PolkadotTrieCursorImpl::makeSearchStateAt(const common::BufferView &key) -> outcome::result { if (trie_->getRoot() == nullptr) { diff --git a/core/storage/trie/polkadot_trie/polkadot_trie_cursor_impl.hpp b/core/storage/trie/polkadot_trie/polkadot_trie_cursor_impl.hpp index 458568da5f..12a718cb42 100644 --- a/core/storage/trie/polkadot_trie/polkadot_trie_cursor_impl.hpp +++ b/core/storage/trie/polkadot_trie/polkadot_trie_cursor_impl.hpp @@ -6,9 +6,11 @@ #pragma once +#include "common/blob.hpp" #include "storage/trie/polkadot_trie/polkadot_trie_cursor.hpp" #include "log/logger.hpp" +#include "storage/trie/polkadot_trie/trie_node.hpp" #include "storage/trie/serialization/polkadot_codec.hpp" namespace kagome::storage::trie { @@ -66,6 +68,8 @@ namespace kagome::storage::trie { [[nodiscard]] std::optional value() const override; + [[nodiscard]] std::optional valueHash() const override; + private: outcome::result seekLowerBoundInternal(const TrieNode ¤t, BufferView left_nibbles); diff --git a/core/storage/trie/polkadot_trie/polkadot_trie_impl.hpp b/core/storage/trie/polkadot_trie/polkadot_trie_impl.hpp index 98022d3b59..8101eb2df3 100644 --- a/core/storage/trie/polkadot_trie/polkadot_trie_impl.hpp +++ b/core/storage/trie/polkadot_trie/polkadot_trie_impl.hpp @@ -6,6 +6,7 @@ #pragma once +#include "common/buffer_view.hpp" #include "storage/trie/polkadot_trie/polkadot_trie.hpp" #include "log/logger.hpp" diff --git a/core/storage/trie/polkadot_trie/trie_node.hpp b/core/storage/trie/polkadot_trie/trie_node.hpp index 0c3a42aca2..b7ca4a4e80 100644 --- a/core/storage/trie/polkadot_trie/trie_node.hpp +++ b/core/storage/trie/polkadot_trie/trie_node.hpp @@ -133,10 +133,14 @@ namespace kagome::storage::trie { bool dirty = true) : hash{hash}, value{std::move(value)}, dirty_{dirty} {} - operator bool() const { + explicit operator bool() const { return is_some(); } + bool operator==(const ValueAndHash &rhs) const { + return std::tie(value, hash) == std::tie(rhs.value, rhs.hash); + } + bool is_none() const { return !is_some(); } diff --git a/core/storage/trie/raw_cursor.hpp b/core/storage/trie/raw_cursor.hpp index 452a3e13dd..90bdcb63d1 100644 --- a/core/storage/trie/raw_cursor.hpp +++ b/core/storage/trie/raw_cursor.hpp @@ -6,10 +6,23 @@ #pragma once +#include #include "storage/trie/child_prefix.hpp" #include "storage/trie/polkadot_trie/trie_node.hpp" namespace kagome::storage::trie { + + enum RawCursorError : uint8_t { + EmptyStack = 1, + ChildBranchNotFound, + StackBackIsNotBranch + }; +} + +OUTCOME_HPP_DECLARE_ERROR(kagome::storage::trie, RawCursorError); + +namespace kagome::storage::trie { + template struct RawCursor { struct Item { @@ -19,7 +32,7 @@ namespace kagome::storage::trie { T t; }; - void update() { + outcome::result update() { child = false; branch_merkle = nullptr; branch_hash.reset(); @@ -27,7 +40,7 @@ namespace kagome::storage::trie { value_hash = nullptr; value_child.reset(); if (stack.empty()) { - return; + return outcome::success(); } auto &item = stack.back(); child = item.child; @@ -40,7 +53,7 @@ namespace kagome::storage::trie { // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-constant-array-index) auto &branch = branches[i]; if (not branch) { - throw std::logic_error{"RawCursor::update branches[branch]=null"}; + return RawCursorError::ChildBranchNotFound; } branch_merkle = &dynamic_cast(*branch).db_key; @@ -56,33 +69,35 @@ namespace kagome::storage::trie { value_child = common::Hash256::fromSpan(*value.value).value(); } } + return outcome::success(); } - void push(Item &&item) { + outcome::result push(Item &&item) { if (not stack.empty() and not stack.back().branch) { - throw std::logic_error{"RawCursor::push branch=None"}; + return RawCursorError::StackBackIsNotBranch; } item.child.match(item.node->getKeyNibbles()); stack.emplace_back(std::move(item)); if (stack.back().branch) { - branchInit(); + OUTCOME_TRY(branchInit()); } else { - update(); + OUTCOME_TRY(update()); } + return outcome::success(); } - void pop() { + outcome::result pop() { stack.pop_back(); - update(); + return update(); } - void branchInit() { - branchNext(false); + outcome::result branchInit() { + return branchNext(false); } - void branchNext(bool next = true) { + outcome::result branchNext(bool next = true) { if (stack.empty()) { - throw std::logic_error{"RawCursor::branchNext top=null"}; + return RawCursorError::EmptyStack; } auto &item = stack.back(); auto &i = item.branch; @@ -103,7 +118,7 @@ namespace kagome::storage::trie { next = false; } } - update(); + return update(); } std::vector stack; diff --git a/core/storage/trie/serialization/trie_serializer_impl.cpp b/core/storage/trie/serialization/trie_serializer_impl.cpp index 2faa0cac32..23285b50ed 100644 --- a/core/storage/trie/serialization/trie_serializer_impl.cpp +++ b/core/storage/trie/serialization/trie_serializer_impl.cpp @@ -17,13 +17,13 @@ namespace kagome::storage::trie { TrieSerializerImpl::TrieSerializerImpl( std::shared_ptr factory, std::shared_ptr codec, - std::shared_ptr node_backend) + std::shared_ptr storage_backend) : trie_factory_{std::move(factory)}, codec_{std::move(codec)}, - node_backend_{std::move(node_backend)} { + storage_backend_{std::move(storage_backend)} { BOOST_ASSERT(trie_factory_ != nullptr); BOOST_ASSERT(codec_ != nullptr); - BOOST_ASSERT(node_backend_ != nullptr); + BOOST_ASSERT(storage_backend_ != nullptr); } RootHash TrieSerializerImpl::getEmptyRootHash() const { @@ -65,32 +65,31 @@ namespace kagome::storage::trie { outcome::result TrieSerializerImpl::storeRootNode( TrieNode &node, StateVersion version) { - auto batch = node_backend_->batch(); + auto batch = storage_backend_->batch(); BOOST_ASSERT(batch != nullptr); - OUTCOME_TRY( - enc, - codec_->encodeNode( - node, - version, - [&](Codec::Visitee visitee) -> outcome::result { - if (auto child_data = std::get_if(&visitee); - child_data != nullptr) { - if (child_data->merkle_value.isHash()) { - return batch->put(child_data->merkle_value.asBuffer(), - std::move(child_data->encoding)); - } - return outcome::success(); // nodes which encoding is shorter - // than its hash are not stored in - // the DB separately - } - auto value_data = std::get(visitee); - // value_data.value is a reference to a buffer stored outside of - // this lambda, so taking its view should be okay - return batch->put(value_data.hash, value_data.value.view()); - })); + auto visitor = [&](Codec::Visitee visitee) -> outcome::result { + if (auto child_data = std::get_if(&visitee); + child_data != nullptr) { + if (child_data->merkle_value.isHash()) { + return batch->put(Space::kTrieNode, + child_data->merkle_value.asBuffer(), + std::move(child_data->encoding)); + } + return outcome::success(); // nodes which encoding is shorter + // than its hash are not stored in + // the DB separately + } + auto value_data = std::get(visitee); + // value_data.value is a reference to a buffer stored outside of + // this lambda, so taking its view should be okay + return batch->put( + Space::kTrieValue, value_data.hash, value_data.value.view()); + }; + + OUTCOME_TRY(enc, codec_->encodeNode(node, version, visitor)); auto hash = codec_->hash256(enc); - OUTCOME_TRY(batch->put(hash, std::move(enc))); + OUTCOME_TRY(batch->put(Space::kTrieNode, hash, std::move(enc))); OUTCOME_TRY(batch->commit()); return hash; @@ -113,7 +112,7 @@ namespace kagome::storage::trie { } BufferOrView enc; if (auto hash = db_key.asHash()) { - BOOST_OUTCOME_TRY(enc, node_backend_->get(*hash)); + BOOST_OUTCOME_TRY(enc, storage_backend_->nodes().get(*hash)); if (on_node_loaded) { on_node_loaded(*hash, enc); } @@ -129,7 +128,7 @@ namespace kagome::storage::trie { outcome::result> TrieSerializerImpl::retrieveValue(const common::Hash256 &hash, const OnNodeLoaded &on_node_loaded) const { - OUTCOME_TRY(value, node_backend_->tryGet(hash)); + OUTCOME_TRY(value, storage_backend_->values().tryGet(hash)); return common::map_optional(std::move(value), [&](common::BufferOrView &&value) { if (on_node_loaded) { diff --git a/core/storage/trie/serialization/trie_serializer_impl.hpp b/core/storage/trie/serialization/trie_serializer_impl.hpp index 3411c8fea7..fd422a8250 100644 --- a/core/storage/trie/serialization/trie_serializer_impl.hpp +++ b/core/storage/trie/serialization/trie_serializer_impl.hpp @@ -24,7 +24,7 @@ namespace kagome::storage::trie { public: TrieSerializerImpl(std::shared_ptr factory, std::shared_ptr codec, - std::shared_ptr node_backend); + std::shared_ptr storage_backend); ~TrieSerializerImpl() override = default; RootHash getEmptyRootHash() const override; @@ -66,6 +66,6 @@ namespace kagome::storage::trie { std::shared_ptr trie_factory_; std::shared_ptr codec_; - std::shared_ptr node_backend_; + std::shared_ptr storage_backend_; }; } // namespace kagome::storage::trie diff --git a/core/storage/trie/trie_batches.hpp b/core/storage/trie/trie_batches.hpp index 6c6b177066..b7ecb7e780 100644 --- a/core/storage/trie/trie_batches.hpp +++ b/core/storage/trie/trie_batches.hpp @@ -6,7 +6,9 @@ #pragma once +#include "common/buffer.hpp" #include "storage/buffer_map_types.hpp" +#include "storage/face/generic_maps.hpp" #include "storage/trie/polkadot_trie/polkadot_trie_cursor.hpp" #include "storage/trie/types.hpp" diff --git a/core/storage/trie/trie_storage_backend.hpp b/core/storage/trie/trie_storage_backend.hpp index 2a893ba1b8..aed7276407 100644 --- a/core/storage/trie/trie_storage_backend.hpp +++ b/core/storage/trie/trie_storage_backend.hpp @@ -15,12 +15,15 @@ namespace kagome::storage::trie { /** - * Adapter for the trie node storage that allows to hide keyspace separation - * along with root hash storing logic from the trie db component + * Provides storage for trie nodes and values. */ - class TrieStorageBackend : public BufferStorage { + class TrieStorageBackend { public: - ~TrieStorageBackend() override = default; + virtual ~TrieStorageBackend() = default; + + virtual BufferStorage &nodes() = 0; + virtual BufferStorage &values() = 0; + virtual std::unique_ptr batch() = 0; }; } // namespace kagome::storage::trie diff --git a/core/storage/trie_pruner/impl/trie_pruner_impl.cpp b/core/storage/trie_pruner/impl/trie_pruner_impl.cpp index ea3da3d5e8..34ba6e54f3 100644 --- a/core/storage/trie_pruner/impl/trie_pruner_impl.cpp +++ b/core/storage/trie_pruner/impl/trie_pruner_impl.cpp @@ -15,6 +15,7 @@ #include "application/app_state_manager.hpp" #include "blockchain/block_tree.hpp" #include "crypto/hasher/hasher_impl.hpp" +#include "storage/buffer_map_types.hpp" #include "storage/database_error.hpp" #include "storage/predefined_keys.hpp" #include "storage/spaced_storage.hpp" @@ -59,20 +60,21 @@ namespace kagome::storage::trie_pruner { TriePrunerImpl::TriePrunerImpl( std::shared_ptr app_state_manager, - std::shared_ptr node_storage, std::shared_ptr serializer, std::shared_ptr codec, std::shared_ptr storage, std::shared_ptr hasher, std::shared_ptr config) - : node_storage_{std::move(node_storage)}, - serializer_{std::move(serializer)}, + : serializer_{std::move(serializer)}, codec_{std::move(codec)}, storage_{std::move(storage)}, + node_storage_{storage_->getSpace(Space::kTrieNode)}, + value_storage_{storage_->getSpace(Space::kTrieValue)}, hasher_{std::move(hasher)}, pruning_depth_{config->statePruningDepth()}, thorough_pruning_{config->enableThoroughPruning()} { BOOST_ASSERT(node_storage_ != nullptr); + BOOST_ASSERT(value_storage_ != nullptr); BOOST_ASSERT(serializer_ != nullptr); BOOST_ASSERT(codec_ != nullptr); BOOST_ASSERT(storage_ != nullptr); @@ -174,9 +176,9 @@ namespace kagome::storage::trie_pruner { outcome::result TriePrunerImpl::pruneFinalized( const primitives::BlockHeader &block) { std::unique_lock lock{mutex_}; - auto node_batch = node_storage_->batch(); - OUTCOME_TRY(prune(*node_batch, block.state_root)); - OUTCOME_TRY(node_batch->commit()); + auto batch = storage_->createBatch(); + OUTCOME_TRY(prune(*batch, block.state_root)); + OUTCOME_TRY(batch->commit()); last_pruned_block_ = block.blockInfo(); OUTCOME_TRY(savePersistentState()); @@ -187,15 +189,13 @@ namespace kagome::storage::trie_pruner { const primitives::BlockHeader &block) { std::unique_lock lock{mutex_}; // should prune even when pruning depth is none - auto node_batch = node_storage_->batch(); - auto value_batch = node_storage_->batch(); - OUTCOME_TRY(prune(*node_batch, block.state_root)); - OUTCOME_TRY(node_batch->commit()); - OUTCOME_TRY(value_batch->commit()); + auto batch = storage_->createBatch(); + OUTCOME_TRY(prune(*batch, block.state_root)); + OUTCOME_TRY(batch->commit()); return outcome::success(); } - outcome::result TriePrunerImpl::prune(BufferBatch &node_batch, + outcome::result TriePrunerImpl::prune(BufferSpacedBatch &batch, const trie::RootHash &root_hash) { auto trie_res = serializer_->retrieveTrie(root_hash, nullptr); if (trie_res.has_error() @@ -216,9 +216,9 @@ namespace kagome::storage::trie_pruner { OUTCOME_TRY( forEachChildTrie(*trie, - [this, &node_batch](common::BufferView child_key, - const trie::RootHash &child_hash) { - return prune(node_batch, child_hash); + [this, &batch](common::BufferView child_key, + const trie::RootHash &child_hash) { + return prune(batch, child_hash); })); size_t nodes_removed = 0; @@ -268,7 +268,7 @@ namespace kagome::storage::trie_pruner { && ref_count == 0) { nodes_removed++; ref_count_.erase(ref_count_it); - OUTCOME_TRY(node_batch.remove(hash)); + OUTCOME_TRY(batch.remove(Space::kTrieNode, hash)); auto hash_opt = node->getValue().hash; if (hash_opt.has_value()) { auto &value_hash = *hash_opt; @@ -279,7 +279,7 @@ namespace kagome::storage::trie_pruner { auto &value_ref_count = value_ref_it->second; value_ref_count--; if (value_ref_count == 0) { - OUTCOME_TRY(node_batch.remove(value_hash)); + OUTCOME_TRY(batch.remove(Space::kTrieValue, value_hash)); value_ref_count_.erase(value_ref_it); values_removed++; } @@ -407,7 +407,9 @@ namespace kagome::storage::trie_pruner { auto value_hash_opt = encoder.getValueHash(*node, version); if (value_hash_opt) { auto &value_ref_count = value_ref_count_[*value_hash_opt]; - OUTCOME_TRY(contains_value, node_storage_->contains(*value_hash_opt)); + OUTCOME_TRY( + contains_value, + storage_->getSpace(Space::kTrieValue)->contains(*value_hash_opt)); if (value_ref_count == 0 && contains_value && !thorough_pruning_) { value_ref_count++; } diff --git a/core/storage/trie_pruner/impl/trie_pruner_impl.hpp b/core/storage/trie_pruner/impl/trie_pruner_impl.hpp index d52ddb9887..9e5e6c5a8d 100644 --- a/core/storage/trie_pruner/impl/trie_pruner_impl.hpp +++ b/core/storage/trie_pruner/impl/trie_pruner_impl.hpp @@ -62,7 +62,6 @@ namespace kagome::storage::trie_pruner { TriePrunerImpl( std::shared_ptr app_state_manager, - std::shared_ptr node_storage, std::shared_ptr serializer, std::shared_ptr codec, std::shared_ptr storage, @@ -119,7 +118,7 @@ namespace kagome::storage::trie_pruner { const primitives::BlockHeader &last_pruned_block, const blockchain::BlockTree &block_tree); - outcome::result prune(BufferBatch &node_batch, + outcome::result prune(BufferSpacedBatch &batch, const storage::trie::RootHash &state); outcome::result addNewStateWith( @@ -134,10 +133,11 @@ namespace kagome::storage::trie_pruner { std::unordered_set immortal_nodes_; std::optional last_pruned_block_; - std::shared_ptr node_storage_; std::shared_ptr serializer_; std::shared_ptr codec_; std::shared_ptr storage_; + std::shared_ptr node_storage_; + std::shared_ptr value_storage_; std::shared_ptr hasher_; const std::optional pruning_depth_{}; diff --git a/core/utils/kagome_db_editor.cpp b/core/utils/kagome_db_editor.cpp index eb3ad29561..45cab3a126 100644 --- a/core/utils/kagome_db_editor.cpp +++ b/core/utils/kagome_db_editor.cpp @@ -4,6 +4,8 @@ * SPDX-License-Identifier: Apache-2.0 */ +#include "storage/buffer_map_types.hpp" +#include "storage/trie/trie_storage_backend.hpp" #if defined(BACKWARD_HAS_BACKTRACE) #include #endif @@ -24,6 +26,7 @@ #include "crypto/hasher/hasher_impl.hpp" #include "network/impl/extrinsic_observer_impl.hpp" #include "runtime/common/runtime_upgrade_tracker_impl.hpp" +#include "storage/face/map_cursor.hpp" #include "storage/predefined_keys.hpp" #include "storage/rocksdb/rocksdb.hpp" #include "storage/trie/impl/trie_storage_backend_impl.hpp" @@ -42,20 +45,16 @@ using common::BufferView; // NOLINTBEGIN(cppcoreguidelines-pro-bounds-pointer-arithmetic) -struct TrieTracker : TrieStorageBackend { - TrieTracker(std::shared_ptr inner) - : inner{std::move(inner)} {} +struct TrieTracker : storage::BufferStorage { + TrieTracker(storage::BufferStorage &inner) : inner{inner} {} std::unique_ptr cursor() override { abort(); } - std::unique_ptr batch() override { - abort(); - } outcome::result get(const BufferView &key) const override { track(key); - return inner->get(key); + return inner.get(key); } outcome::result> tryGet( const BufferView &key) const override { @@ -69,6 +68,7 @@ struct TrieTracker : TrieStorageBackend { BufferOrView &&value) override { abort(); } + outcome::result remove(const common::BufferView &key) override { abort(); } @@ -80,10 +80,31 @@ struct TrieTracker : TrieStorageBackend { return keys.contains(common::Hash256::fromSpan(key).value()); } - std::shared_ptr inner; + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + storage::BufferStorage &inner; mutable std::set keys; }; +struct TrieTrackerBackend : TrieStorageBackend { + TrieTrackerBackend(std::shared_ptr backend) + : backend{std::move(backend)}, node_tracker{backend->nodes()} {} + + storage::BufferStorage &nodes() override { + return node_tracker; + } + + storage::BufferStorage &values() override { + return backend->values(); + } + + std::unique_ptr batch() override { + return backend->batch(); + } + + std::shared_ptr backend; + TrieTracker node_tracker; +}; + template using sptr = std::shared_ptr; @@ -238,11 +259,11 @@ int db_editor_main(int argc, const char **argv) { auto factory = std::make_shared(); std::shared_ptr storage; - std::shared_ptr buffer_storage; + std::shared_ptr buffer_storage; try { storage = storage::RocksDb::create(argv[DB_PATH], rocksdb::Options()).value(); - storage->dropColumn(storage::Space::kBlockBody); + storage->dropColumn(storage::Space::kBlockBody).value(); buffer_storage = storage->getSpace(storage::Space::kDefault); } catch (std::system_error &e) { log->error("{}", e.what()); @@ -250,7 +271,7 @@ int db_editor_main(int argc, const char **argv) { return 0; } - auto trie_node_tracker = std::make_shared( + auto trie_node_tracker = std::make_shared( std::make_shared(storage)); auto injector = di::make_injector( @@ -409,7 +430,7 @@ int db_editor_main(int argc, const char **argv) { TicToc t2("Process DB.", log); while (db_cursor->isValid() && db_cursor->key().has_value()) { auto key = db_cursor->key().value(); - if (tracker->tracked(key)) { + if (tracker->node_tracker.tracked(key)) { db_cursor->next().value(); continue; } diff --git a/core/utils/map_entry.hpp b/core/utils/map_entry.hpp index 8d8070b1da..d3dd2ab307 100644 --- a/core/utils/map_entry.hpp +++ b/core/utils/map_entry.hpp @@ -73,4 +73,4 @@ namespace kagome { auto entry(std::unordered_map &map, const K &key) { return MapEntry>{map, key}; } -} // namespace kagome \ No newline at end of file +} // namespace kagome diff --git a/test/core/consensus/CMakeLists.txt b/test/core/consensus/CMakeLists.txt index f71417bcd5..d5f82b53d9 100644 --- a/test/core/consensus/CMakeLists.txt +++ b/test/core/consensus/CMakeLists.txt @@ -14,4 +14,5 @@ addtest(beefy_test target_link_libraries(beefy_test logger_for_tests network + in_memory_storage ) diff --git a/test/core/consensus/beefy_test.cpp b/test/core/consensus/beefy_test.cpp index 7b8498c281..a15824ce0d 100644 --- a/test/core/consensus/beefy_test.cpp +++ b/test/core/consensus/beefy_test.cpp @@ -26,9 +26,9 @@ #include "mock/core/runtime/beefy_api.hpp" #include "network/impl/protocols/beefy_protocol_impl.hpp" #include "primitives/event_types.hpp" -#include "storage/in_memory/in_memory_spaced_storage.hpp" #include "testutil/lazy.hpp" #include "testutil/prepare_loggers.hpp" +#include "testutil/storage/in_memory/in_memory_spaced_storage.hpp" using kagome::beefyMmrDigest; using kagome::TestThreadPool; diff --git a/test/core/consensus/grandpa/CMakeLists.txt b/test/core/consensus/grandpa/CMakeLists.txt index 5fbb7489dd..521c9cbaaa 100644 --- a/test/core/consensus/grandpa/CMakeLists.txt +++ b/test/core/consensus/grandpa/CMakeLists.txt @@ -45,4 +45,5 @@ target_link_libraries(authority_manager_test dummy_error logger_for_tests storage + in_memory_storage ) diff --git a/test/core/consensus/grandpa/authority_manager_test.cpp b/test/core/consensus/grandpa/authority_manager_test.cpp index 00337d2f62..975d68bf10 100644 --- a/test/core/consensus/grandpa/authority_manager_test.cpp +++ b/test/core/consensus/grandpa/authority_manager_test.cpp @@ -12,7 +12,7 @@ #include "mock/core/application/app_state_manager_mock.hpp" #include "mock/core/blockchain/block_tree_mock.hpp" #include "mock/core/runtime/grandpa_api_mock.hpp" -#include "storage/in_memory/in_memory_spaced_storage.hpp" +#include "testutil/storage/in_memory/in_memory_spaced_storage.hpp" #include "testutil/prepare_loggers.hpp" diff --git a/test/core/consensus/timeline/CMakeLists.txt b/test/core/consensus/timeline/CMakeLists.txt index 11e8e94da1..7de7fd975b 100644 --- a/test/core/consensus/timeline/CMakeLists.txt +++ b/test/core/consensus/timeline/CMakeLists.txt @@ -22,6 +22,7 @@ target_link_libraries(slots_util_test clock storage logger_for_tests + in_memory_storage ) addtest(timeline_test diff --git a/test/core/consensus/timeline/slots_util_test.cpp b/test/core/consensus/timeline/slots_util_test.cpp index 05c4046ce0..3b558464af 100644 --- a/test/core/consensus/timeline/slots_util_test.cpp +++ b/test/core/consensus/timeline/slots_util_test.cpp @@ -15,8 +15,8 @@ #include "mock/core/runtime/babe_api_mock.hpp" #include "mock/core/storage/spaced_storage_mock.hpp" #include "mock/core/storage/trie/trie_storage_mock.hpp" -#include "storage/in_memory/in_memory_storage.hpp" #include "testutil/prepare_loggers.hpp" +#include "testutil/storage/in_memory/in_memory_storage.hpp" using namespace kagome; using application::AppStateManagerMock; diff --git a/test/core/network/CMakeLists.txt b/test/core/network/CMakeLists.txt index 25924cfb60..984754dcba 100644 --- a/test/core/network/CMakeLists.txt +++ b/test/core/network/CMakeLists.txt @@ -36,6 +36,7 @@ target_link_libraries(state_protocol_observer_test logger_for_tests storage network + in_memory_storage ) addtest(sync_protocol_observer_test diff --git a/test/core/network/state_protocol_observer_test.cpp b/test/core/network/state_protocol_observer_test.cpp index 7c2693fdfb..1087207fd5 100644 --- a/test/core/network/state_protocol_observer_test.cpp +++ b/test/core/network/state_protocol_observer_test.cpp @@ -12,7 +12,6 @@ #include "mock/core/blockchain/block_header_repository_mock.hpp" #include "mock/core/storage/trie_pruner/trie_pruner_mock.hpp" #include "network/types/state_request.hpp" -#include "storage/in_memory/in_memory_spaced_storage.hpp" #include "storage/trie/impl/trie_storage_backend_impl.hpp" #include "storage/trie/impl/trie_storage_impl.hpp" #include "storage/trie/polkadot_trie/polkadot_trie_factory_impl.hpp" @@ -23,6 +22,7 @@ #include "testutil/literals.hpp" #include "testutil/outcome.hpp" #include "testutil/prepare_loggers.hpp" +#include "testutil/storage/in_memory/in_memory_spaced_storage.hpp" using namespace kagome; diff --git a/test/core/runtime/CMakeLists.txt b/test/core/runtime/CMakeLists.txt index 93e349cde5..290486c4ed 100644 --- a/test/core/runtime/CMakeLists.txt +++ b/test/core/runtime/CMakeLists.txt @@ -37,6 +37,7 @@ target_link_libraries(trie_storage_provider_test trie_storage_provider storage logger_for_tests + in_memory_storage ) addtest(uncompress_code_test @@ -85,6 +86,7 @@ target_link_libraries(runtime_upgrade_tracker_test logger log_configurator scale::scale + in_memory_storage ) addtest(instance_pool_test instance_pool_test.cpp) diff --git a/test/core/runtime/binaryen/CMakeLists.txt b/test/core/runtime/binaryen/CMakeLists.txt index 6f65ecf3bb..1f54c1a350 100644 --- a/test/core/runtime/binaryen/CMakeLists.txt +++ b/test/core/runtime/binaryen/CMakeLists.txt @@ -37,6 +37,7 @@ target_link_libraries(tagged_transaction_queue_test core_api_factory logger_for_tests filesystem + in_memory_storage ) addtest(block_builder_api_test @@ -49,6 +50,7 @@ target_link_libraries(block_builder_api_test core_api filesystem logger_for_tests + in_memory_storage ) addtest(binaryen_parachain_test @@ -64,6 +66,7 @@ target_link_libraries(binaryen_parachain_test hasher pbkdf2_provider network + in_memory_storage logger_for_tests ) @@ -78,6 +81,7 @@ target_link_libraries(metadata_test core_api core_api_factory logger_for_tests + in_memory_storage ) addtest(runtime_external_interface_test diff --git a/test/core/runtime/runtime_test_base.hpp b/test/core/runtime/runtime_test_base.hpp index a2429c6221..08c37f7284 100644 --- a/test/core/runtime/runtime_test_base.hpp +++ b/test/core/runtime/runtime_test_base.hpp @@ -49,10 +49,10 @@ #include "runtime/module.hpp" #include "runtime/runtime_context.hpp" #include "runtime/wabt/instrument.hpp" -#include "storage/in_memory/in_memory_storage.hpp" #include "testutil/literals.hpp" #include "testutil/outcome.hpp" #include "testutil/runtime/common/basic_code_provider.hpp" +#include "testutil/storage/in_memory/in_memory_storage.hpp" using kagome::application::AppConfigurationMock; using kagome::runtime::RuntimeInstancesPoolImpl; diff --git a/test/core/runtime/runtime_upgrade_tracker_test.cpp b/test/core/runtime/runtime_upgrade_tracker_test.cpp index 0800720d20..ca9435c7c7 100644 --- a/test/core/runtime/runtime_upgrade_tracker_test.cpp +++ b/test/core/runtime/runtime_upgrade_tracker_test.cpp @@ -12,10 +12,11 @@ #include "mock/core/blockchain/block_storage_mock.hpp" #include "mock/core/blockchain/block_tree_mock.hpp" #include "mock/core/storage/spaced_storage_mock.hpp" -#include "storage/in_memory/in_memory_storage.hpp" +#include "storage/buffer_map_types.hpp" #include "testutil/literals.hpp" #include "testutil/outcome.hpp" #include "testutil/prepare_loggers.hpp" +#include "testutil/storage/in_memory/in_memory_storage.hpp" using kagome::common::Hash256; using std::string_literals::operator""s; @@ -85,7 +86,7 @@ class RuntimeUpgradeTrackerTest : public testing::Test { std::shared_ptr block_tree_; std::shared_ptr sub_engine_; - std::shared_ptr buffer_storage_; + std::shared_ptr buffer_storage_; std::shared_ptr storage_; std::shared_ptr diff --git a/test/core/runtime/trie_storage_provider_test.cpp b/test/core/runtime/trie_storage_provider_test.cpp index 20429dfa56..bbd8618d1e 100644 --- a/test/core/runtime/trie_storage_provider_test.cpp +++ b/test/core/runtime/trie_storage_provider_test.cpp @@ -11,7 +11,6 @@ #include "common/buffer.hpp" #include "mock/core/storage/trie_pruner/trie_pruner_mock.hpp" #include "runtime/common/runtime_execution_error.hpp" -#include "storage/in_memory/in_memory_spaced_storage.hpp" #include "storage/trie/impl/trie_storage_backend_impl.hpp" #include "storage/trie/impl/trie_storage_impl.hpp" #include "storage/trie/polkadot_trie/polkadot_trie_factory_impl.hpp" @@ -20,6 +19,7 @@ #include "testutil/literals.hpp" #include "testutil/outcome.hpp" #include "testutil/prepare_loggers.hpp" +#include "testutil/storage/in_memory/in_memory_spaced_storage.hpp" using kagome::common::Buffer; using kagome::runtime::RuntimeExecutionError; diff --git a/test/core/runtime/wavm/CMakeLists.txt b/test/core/runtime/wavm/CMakeLists.txt index 88ba02c971..8b791f22bc 100644 --- a/test/core/runtime/wavm/CMakeLists.txt +++ b/test/core/runtime/wavm/CMakeLists.txt @@ -42,4 +42,5 @@ target_link_libraries(core_integration_test storage filesystem logger_for_tests + in_memory_storage ) diff --git a/test/core/storage/changes_trie/CMakeLists.txt b/test/core/storage/changes_trie/CMakeLists.txt index 3d4fb48fac..713b42cd26 100644 --- a/test/core/storage/changes_trie/CMakeLists.txt +++ b/test/core/storage/changes_trie/CMakeLists.txt @@ -10,5 +10,6 @@ addtest(changes_tracker_test target_link_libraries(changes_tracker_test storage logger_for_tests + in_memory_storage ) diff --git a/test/core/storage/changes_trie/changes_tracker_test.cpp b/test/core/storage/changes_trie/changes_tracker_test.cpp index b7165ed51d..269c2aa4c2 100644 --- a/test/core/storage/changes_trie/changes_tracker_test.cpp +++ b/test/core/storage/changes_trie/changes_tracker_test.cpp @@ -12,8 +12,6 @@ #include "mock/core/storage/trie_pruner/trie_pruner_mock.hpp" #include "primitives/event_types.hpp" #include "scale/scale.hpp" -#include "storage/in_memory/in_memory_spaced_storage.hpp" -#include "storage/in_memory/in_memory_storage.hpp" #include "storage/trie/impl/persistent_trie_batch_impl.hpp" #include "storage/trie/impl/trie_storage_backend_impl.hpp" #include "storage/trie/polkadot_trie/polkadot_trie_factory_impl.hpp" @@ -22,6 +20,8 @@ #include "testutil/literals.hpp" #include "testutil/outcome.hpp" #include "testutil/prepare_loggers.hpp" +#include "testutil/storage/in_memory/in_memory_spaced_storage.hpp" +#include "testutil/storage/in_memory/in_memory_storage.hpp" using kagome::api::Session; using kagome::blockchain::BlockHeaderRepositoryMock; diff --git a/test/core/storage/trie/polkadot_trie/polkadot_trie_test.cpp b/test/core/storage/trie/polkadot_trie/polkadot_trie_test.cpp index 022d5ff90b..f2ff8f08bd 100644 --- a/test/core/storage/trie/polkadot_trie/polkadot_trie_test.cpp +++ b/test/core/storage/trie/polkadot_trie/polkadot_trie_test.cpp @@ -6,12 +6,12 @@ #include -#include "storage/in_memory/in_memory_storage.hpp" #include "storage/trie/polkadot_trie/polkadot_trie_impl.hpp" #include "storage/trie/polkadot_trie/trie_error.hpp" #include "testutil/literals.hpp" #include "testutil/outcome.hpp" #include "testutil/prepare_loggers.hpp" +#include "testutil/storage/in_memory/in_memory_storage.hpp" #include "testutil/storage/polkadot_trie_printer.hpp" using kagome::common::Buffer; diff --git a/test/core/storage/trie/polkadot_trie_cursor_dummy.hpp b/test/core/storage/trie/polkadot_trie_cursor_dummy.hpp index 6f4eb97246..a9c38398db 100644 --- a/test/core/storage/trie/polkadot_trie_cursor_dummy.hpp +++ b/test/core/storage/trie/polkadot_trie_cursor_dummy.hpp @@ -6,7 +6,10 @@ #pragma once +#include "common/blob.hpp" +#include "crypto/blake2/blake2b.h" #include "storage/trie/polkadot_trie/polkadot_trie_cursor.hpp" +#include "storage/trie/polkadot_trie/trie_node.hpp" namespace kagome::storage::trie { @@ -14,7 +17,7 @@ namespace kagome::storage::trie { class PolkadotTrieCursorDummy : public PolkadotTrieCursor { private: std::map> key_val_; - decltype(key_val_.begin()) current_; + decltype(key_val_)::iterator current_; public: explicit PolkadotTrieCursorDummy( @@ -69,5 +72,9 @@ namespace kagome::storage::trie { std::optional value() const override { return BufferView{current_->second}; } + + std::optional valueHash() const override { + return ValueHash{crypto::blake2b<32>(current_->second), false}; + } }; } // namespace kagome::storage::trie diff --git a/test/core/storage/trie/trie_storage/CMakeLists.txt b/test/core/storage/trie/trie_storage/CMakeLists.txt index a5e0b265d5..605afc4e5b 100644 --- a/test/core/storage/trie/trie_storage/CMakeLists.txt +++ b/test/core/storage/trie/trie_storage/CMakeLists.txt @@ -18,12 +18,5 @@ target_link_libraries(polkadot_trie_storage_test Boost::boost base_rocksdb_test logger_for_tests - ) - -addtest(trie_storage_backend_test - trie_storage_backend_test.cpp - ) -target_link_libraries(trie_storage_backend_test - storage - blob + in_memory_storage ) diff --git a/test/core/storage/trie/trie_storage/polkadot_codec_node_decoding_test.cpp b/test/core/storage/trie/trie_storage/polkadot_codec_node_decoding_test.cpp index ce286d72c7..17e8366c60 100644 --- a/test/core/storage/trie/trie_storage/polkadot_codec_node_decoding_test.cpp +++ b/test/core/storage/trie/trie_storage/polkadot_codec_node_decoding_test.cpp @@ -7,6 +7,7 @@ #include #include + #include "storage/trie/polkadot_trie/trie_node.hpp" #include "storage/trie/serialization/buffer_stream.hpp" #include "storage/trie/serialization/polkadot_codec.hpp" @@ -30,7 +31,7 @@ TEST_P(NodeDecodingTest, GetHeader) { EXPECT_OUTCOME_TRUE( encoded, codec->encodeNode(*node, storage::trie::StateVersion::V0, {})); EXPECT_OUTCOME_TRUE(decoded, codec->decodeNode(encoded)); - auto decoded_node = std::dynamic_pointer_cast(decoded); + auto decoded_node = decoded; EXPECT_EQ(decoded_node->getKeyNibbles(), node->getKeyNibbles()); EXPECT_EQ(decoded_node->getValue(), node->getValue()); } diff --git a/test/core/storage/trie/trie_storage/trie_batch_test.cpp b/test/core/storage/trie/trie_storage/trie_batch_test.cpp index 2954b136b9..637f3eea8b 100644 --- a/test/core/storage/trie/trie_storage/trie_batch_test.cpp +++ b/test/core/storage/trie/trie_storage/trie_batch_test.cpp @@ -6,11 +6,14 @@ #include #include +#include +#include "gmock/gmock.h" #include "mock/core/storage/spaced_storage_mock.hpp" +#include "mock/core/storage/trie/trie_batches_mock.hpp" #include "mock/core/storage/trie_pruner/trie_pruner_mock.hpp" +#include "mock/core/storage/write_batch_mock.hpp" #include "storage/changes_trie/impl/storage_changes_tracker_impl.hpp" -#include "storage/in_memory/in_memory_storage.hpp" #include "storage/trie/impl/topper_trie_batch_impl.hpp" #include "storage/trie/impl/trie_storage_backend_impl.hpp" #include "storage/trie/impl/trie_storage_impl.hpp" @@ -21,6 +24,8 @@ #include "testutil/literals.hpp" #include "testutil/outcome.hpp" #include "testutil/storage/base_rocksdb_test.hpp" +#include "testutil/storage/in_memory/in_memory_batch.hpp" +#include "testutil/storage/in_memory/in_memory_storage.hpp" using namespace kagome::storage::trie; using kagome::api::Session; @@ -205,6 +210,9 @@ TEST_F(TrieBatchTest, ConsistentOnFailure) { auto spaced_db = std::make_shared(); ON_CALL(*spaced_db, getSpace(Space::kTrieNode)).WillByDefault(Return(db)); ON_CALL(*spaced_db, getSpace(Space::kTrieValue)).WillByDefault(Return(db)); + ON_CALL(*spaced_db, createBatch()).WillByDefault(Invoke([&spaced_db]() { + return std::make_unique(*spaced_db); + })); auto factory = std::make_shared(); auto codec = std::make_shared(); diff --git a/test/core/storage/trie/trie_storage/trie_storage_backend_test.cpp b/test/core/storage/trie/trie_storage/trie_storage_backend_test.cpp deleted file mode 100644 index 9c9598cd4a..0000000000 --- a/test/core/storage/trie/trie_storage/trie_storage_backend_test.cpp +++ /dev/null @@ -1,93 +0,0 @@ -/** - * Copyright Quadrivium LLC - * All Rights Reserved - * SPDX-License-Identifier: Apache-2.0 - */ - -#include - -#include - -#include "mock/core/storage/generic_storage_mock.hpp" -#include "mock/core/storage/spaced_storage_mock.hpp" -#include "mock/core/storage/write_batch_mock.hpp" -#include "storage/trie/impl/trie_storage_backend_impl.hpp" -#include "testutil/literals.hpp" -#include "testutil/outcome.hpp" - -using kagome::common::Buffer; -using kagome::common::BufferView; -using kagome::storage::BufferStorageMock; -using kagome::storage::SpacedStorageMock; -using kagome::storage::face::WriteBatchMock; -using kagome::storage::trie::TrieStorageBackendImpl; -using testing::Invoke; -using testing::Return; - -class TrieDbBackendTest : public testing::Test { - public: - void SetUp() override { - ON_CALL(*spaced_storage, getSpace(kagome::storage::Space::kTrieNode)) - .WillByDefault(Return(storage)); - backend = std::make_unique(spaced_storage); - } - - std::shared_ptr storage = - std::make_shared(); - std::shared_ptr spaced_storage = - std::make_shared(); - std::unique_ptr backend; -}; - -/** - * @given trie backend - * @when put a value to it - * @then it puts a prefixed value to the storage - */ -TEST_F(TrieDbBackendTest, Put) { - auto key = "abc"_buf; - ((*storage).gmock_put(BufferView{key}, "123"_buf))( - ::testing::internal::GetWithoutMatchers(), nullptr) - .InternalExpectedAt( - "_file_name_", 40, "*storage", "put(prefixed, \"123\"_buf)") - .WillOnce(Return(outcome::success())); - EXPECT_OUTCOME_TRUE_1(backend->put("abc"_buf, "123"_buf)); -} - -/** - * @given trie backend - * @when get a value from it - * @then it takes a prefixed value from the storage - */ -TEST_F(TrieDbBackendTest, Get) { - auto key = "abc"_buf; - EXPECT_CALL(*storage, getMock(BufferView{key})).WillOnce(Return("123"_buf)); - EXPECT_OUTCOME_TRUE_1(backend->get("abc"_buf)); -} - -/** - * @given trie backend batch - * @when perform operations on it - * @then it delegates them to the underlying storage batch with added prefixes - */ -TEST_F(TrieDbBackendTest, Batch) { - auto batch_mock = std::make_unique>(); - auto buf_abc = "abc"_buf; - EXPECT_CALL(*batch_mock, put(buf_abc.view(), "123"_buf)) - .WillOnce(Return(outcome::success())); - auto buf_def = "def"_buf; - EXPECT_CALL(*batch_mock, put(buf_def.view(), "123"_buf)) - .WillOnce(Return(outcome::success())); - EXPECT_CALL(*batch_mock, remove(buf_abc.view())) - .WillOnce(Return(outcome::success())); - EXPECT_CALL(*batch_mock, commit()).WillOnce(Return(outcome::success())); - - EXPECT_CALL(*storage, batch()) - .WillOnce(Return(testing::ByMove(std::move(batch_mock)))); - - auto batch = backend->batch(); - EXPECT_OUTCOME_TRUE_1(batch->put("abc"_buf, "123"_buf)); - EXPECT_OUTCOME_TRUE_1(batch->put("def"_buf, "123"_buf)); - EXPECT_OUTCOME_TRUE_1(batch->remove("abc"_buf)); - EXPECT_OUTCOME_TRUE_1(batch->commit()); -} diff --git a/test/core/storage/trie_pruner/trie_pruner_test.cpp b/test/core/storage/trie_pruner/trie_pruner_test.cpp index 22ee018a46..885e3956bf 100644 --- a/test/core/storage/trie_pruner/trie_pruner_test.cpp +++ b/test/core/storage/trie_pruner/trie_pruner_test.cpp @@ -20,7 +20,10 @@ #include "mock/core/storage/trie/serialization/trie_serializer_mock.hpp" #include "mock/core/storage/trie/trie_storage_backend_mock.hpp" #include "mock/core/storage/write_batch_mock.hpp" +#include "storage/buffer_map_types.hpp" #include "storage/database_error.hpp" +#include "storage/spaces.hpp" +#include "storage/trie/impl/trie_storage_backend_impl.hpp" #include "storage/trie/polkadot_trie/polkadot_trie_factory_impl.hpp" #include "storage/trie/polkadot_trie/polkadot_trie_impl.hpp" #include "storage/trie/serialization/polkadot_codec.hpp" @@ -164,8 +167,7 @@ class TriePrunerTest : public testing::Test { ON_CALL(*config_mock, statePruningDepth()).WillByDefault(Return(16)); ON_CALL(*config_mock, enableThoroughPruning()).WillByDefault(Return(true)); - trie_node_storage_mock.reset( - new testing::NiceMock()); + trie_node_storage_mock.reset(new testing::NiceMock()); persistent_storage_mock.reset( new testing::NiceMock); serializer_mock.reset(new testing::NiceMock); @@ -184,10 +186,13 @@ class TriePrunerTest : public testing::Test { ON_CALL(*persistent_storage_mock, getSpace(kDefault)) .WillByDefault(Invoke([this](auto) { return pruner_space; })); + ON_CALL(*persistent_storage_mock, getSpace(kTrieNode)) + .WillByDefault(Invoke([this](auto) { return trie_node_storage_mock; })); + ON_CALL(*persistent_storage_mock, getSpace(kTrieValue)) + .WillByDefault(Invoke([this](auto) { return trie_node_storage_mock; })); pruner.reset(new TriePrunerImpl( std::make_shared(), - trie_node_storage_mock, serializer_mock, codec_mock, persistent_storage_mock, @@ -213,7 +218,6 @@ class TriePrunerTest : public testing::Test { pruner.reset(new TriePrunerImpl( std::make_shared(), - trie_node_storage_mock, serializer_mock, codec_mock, persistent_storage_mock, @@ -258,7 +262,7 @@ class TriePrunerTest : public testing::Test { std::unique_ptr pruner; std::shared_ptr serializer_mock; - std::shared_ptr trie_node_storage_mock; + std::shared_ptr trie_node_storage_mock; std::shared_ptr> persistent_storage_mock; std::shared_ptr codec_mock; std::shared_ptr hasher; @@ -348,12 +352,15 @@ TEST_F(TriePrunerTest, BasicScenario) { {{"_0"_hash256, makeTransparentNode({NODE, "_0"_hash256, {}})}, {"_5"_hash256, makeTransparentNode({NODE, "_5"_hash256, {}})}}})); - EXPECT_CALL(*trie_node_storage_mock, batch()).WillRepeatedly(Invoke([]() { - auto batch = std::make_unique>(); - EXPECT_CALL(*batch, remove(_)).WillRepeatedly(Return(outcome::success())); - EXPECT_CALL(*batch, commit()).WillOnce(Return(outcome::success())); - return batch; - })); + EXPECT_CALL(*persistent_storage_mock, createBatch()) + .WillRepeatedly(Invoke([]() { + auto batch = + std::make_unique>(); + EXPECT_CALL(*batch, remove(_, _)) + .WillRepeatedly(Return(outcome::success())); + EXPECT_CALL(*batch, commit()).WillOnce(Return(outcome::success())); + return batch; + })); EXPECT_CALL(*serializer_mock, retrieveTrie("root1"_hash256, _)) .WillOnce(testing::Return(trie)); BlockHeader header1{.number = 1, .state_root = "root1"_hash256}; @@ -469,7 +476,7 @@ TEST_F(TriePrunerTest, RandomTree) { std::map node_storage; std::set inserted_keys; - EXPECT_CALL(*trie_node_storage_mock, get(_)) + EXPECT_CALL(*trie_node_storage_mock, getMock(_)) .WillRepeatedly( Invoke([&node_storage](auto &k) -> outcome::result { auto it = node_storage.find(k); @@ -480,7 +487,9 @@ TEST_F(TriePrunerTest, RandomTree) { })); trie::TrieSerializerImpl serializer{ - trie_factory, codec, trie_node_storage_mock}; + trie_factory, + codec, + std::make_shared(persistent_storage_mock)}; std::vector> kv; std::mt19937 rand; rand.seed(42); @@ -500,21 +509,27 @@ TEST_F(TriePrunerTest, RandomTree) { return serializer.retrieveNode(node, nullptr); })); - for (unsigned i = 0; i < STATES_NUM; i++) { - EXPECT_CALL(*trie_node_storage_mock, batch()) - .WillOnce(Invoke([&node_storage]() { - auto batch_mock = - std::make_unique>(); - EXPECT_CALL(*batch_mock, put(_, _)) - .WillRepeatedly(Invoke([&node_storage](auto &k, auto &v) { - node_storage[k] = v; - return outcome::success(); - })); - EXPECT_CALL(*batch_mock, commit()) - .WillRepeatedly(Return(outcome::success())); - return batch_mock; - })); + EXPECT_CALL(*persistent_storage_mock, createBatch()) + .WillRepeatedly(Invoke([&node_storage]() { + auto batch_mock = + std::make_unique>(); + EXPECT_CALL(*batch_mock, put(Space::kTrieNode, _, _)) + .WillRepeatedly(Invoke([&node_storage](Space, auto &k, auto &v) { + node_storage[k] = v; + return outcome::success(); + })); + EXPECT_CALL(*batch_mock, remove(Space::kTrieNode, _)) + .WillRepeatedly(Invoke([&node_storage](Space, auto &k) { + node_storage.erase(k); + return outcome::success(); + })); + + EXPECT_CALL(*batch_mock, commit()) + .WillRepeatedly(Return(outcome::success())); + return batch_mock; + })); + for (unsigned i = 0; i < STATES_NUM; i++) { for (unsigned j = 0; j < INSERT_PER_STATE; j++) { auto k = randomBuffer(rand); inserted_keys.insert(k); @@ -551,19 +566,6 @@ TEST_F(TriePrunerTest, RandomTree) { roots.push_back(root); if (i >= 16) { - EXPECT_CALL(*trie_node_storage_mock, batch()) - .WillOnce(Invoke([&node_storage]() { - auto batch = - std::make_unique>(); - EXPECT_CALL(*batch, remove(_)) - .WillRepeatedly(Invoke([&node_storage](auto &k) { - node_storage.erase(k); - return outcome::success(); - })); - EXPECT_CALL(*batch, commit()).WillOnce(Return(outcome::success())); - return batch; - })); - const auto &root = roots[i - 16]; BlockHeader header{.number = i - 16, .state_root = root}; @@ -572,11 +574,12 @@ TEST_F(TriePrunerTest, RandomTree) { } } for (unsigned i = STATES_NUM - 16; i < STATES_NUM; i++) { - EXPECT_CALL(*trie_node_storage_mock, batch()) + EXPECT_CALL(*persistent_storage_mock, createBatch()) .WillOnce(Invoke([&node_storage]() { - auto batch = std::make_unique>(); - EXPECT_CALL(*batch, remove(_)) - .WillRepeatedly(Invoke([&node_storage](auto &k) { + auto batch = + std::make_unique>(); + EXPECT_CALL(*batch, remove(Space::kTrieNode, _)) + .WillRepeatedly(Invoke([&node_storage](Space, auto &k) { node_storage.erase(k); return outcome::success(); })); @@ -706,7 +709,7 @@ TEST_F(TriePrunerTest, FastSyncScenario) { auto block_tree = std::make_shared>(); - ON_CALL(*trie_node_storage_mock, get(_)) + ON_CALL(*trie_node_storage_mock, getMock(_)) .WillByDefault(Invoke( [&](auto &key) -> outcome::result { if (node_storage.count(key) == 0) { @@ -715,18 +718,19 @@ TEST_F(TriePrunerTest, FastSyncScenario) { return kagome::common::BufferOrView{node_storage.at(key).view()}; })); - ON_CALL(*trie_node_storage_mock, batch()).WillByDefault(Invoke([&]() { + ON_CALL(*persistent_storage_mock, createBatch()).WillByDefault(Invoke([&]() { auto batch = std::make_unique< - testing::NiceMock>>(); - ON_CALL(*batch, put(_, _)) - .WillByDefault(Invoke([&](auto &key, auto &value) { + testing::NiceMock>>(); + ON_CALL(*batch, put(Space::kTrieNode, _, _)) + .WillByDefault(Invoke([&](Space, auto &key, auto &value) { node_storage[key] = value; return outcome::success(); })); - ON_CALL(*batch, remove(_)).WillByDefault(Invoke([&](auto &key) { - node_storage.erase(key); - return outcome::success(); - })); + ON_CALL(*batch, remove(Space::kTrieNode, _)) + .WillByDefault(Invoke([&](Space, auto &key) { + node_storage.erase(key); + return outcome::success(); + })); ON_CALL(*batch, commit()).WillByDefault(Return(outcome::success())); return batch; })); @@ -744,7 +748,9 @@ TEST_F(TriePrunerTest, FastSyncScenario) { .value()); trie::TrieSerializerImpl serializer{ - trie_factory, codec, trie_node_storage_mock}; + trie_factory, + codec, + std::make_shared(persistent_storage_mock)}; ON_CALL(*serializer_mock, retrieveTrie(genesis_state_root, _)) .WillByDefault(Return(genesis_trie)); diff --git a/test/external-project-test/src/main.cpp b/test/external-project-test/src/main.cpp index c785d9af89..ff5c8c3206 100644 --- a/test/external-project-test/src/main.cpp +++ b/test/external-project-test/src/main.cpp @@ -97,13 +97,7 @@ int main() { auto state_pruner = std::make_shared( - app_state_manager, - node_storage_backend, - serializer, - codec, - database, - hasher, - config); + app_state_manager, serializer, codec, database, hasher, config); std::shared_ptr trie_storage = kagome::storage::trie::TrieStorageImpl::createEmpty( diff --git a/test/mock/core/storage/generic_storage_mock.hpp b/test/mock/core/storage/generic_storage_mock.hpp index 55fd3b1ab8..96462d092b 100644 --- a/test/mock/core/storage/generic_storage_mock.hpp +++ b/test/mock/core/storage/generic_storage_mock.hpp @@ -13,12 +13,15 @@ namespace kagome::storage::face { template - struct GenericStorageMock : public GenericStorage { + struct GenericStorageMock : public BatchableStorage { MOCK_METHOD0_T(batch, std::unique_ptr>()); MOCK_METHOD0_T(cursor, std::unique_ptr>()); - MOCK_METHOD(outcome::result, getMock, (const View &), (const)); + MOCK_METHOD(outcome::result>, + getMock, + (const View &), + (const)); MOCK_METHOD(outcome::result>, tryGetMock, diff --git a/test/mock/core/storage/spaced_storage_mock.hpp b/test/mock/core/storage/spaced_storage_mock.hpp index 02ec4f789c..4355ffaccf 100644 --- a/test/mock/core/storage/spaced_storage_mock.hpp +++ b/test/mock/core/storage/spaced_storage_mock.hpp @@ -14,7 +14,15 @@ namespace kagome::storage { class SpacedStorageMock : public SpacedStorage { public: - MOCK_METHOD(std::shared_ptr, getSpace, (Space), (override)); + MOCK_METHOD(std::shared_ptr, + getSpace, + (Space), + (override)); + + MOCK_METHOD(std::unique_ptr, + createBatch, + (), + (override)); }; } // namespace kagome::storage diff --git a/test/mock/core/storage/trie/polkadot_trie_cursor_mock.h b/test/mock/core/storage/trie/polkadot_trie_cursor_mock.h index 38619a7645..f0f7fe3c57 100644 --- a/test/mock/core/storage/trie/polkadot_trie_cursor_mock.h +++ b/test/mock/core/storage/trie/polkadot_trie_cursor_mock.h @@ -7,6 +7,7 @@ #ifndef KAGOME_POLKADOT_TRIE_CURSOR_MOCK_H #define KAGOME_POLKADOT_TRIE_CURSOR_MOCK_H +#include "common/blob.hpp" #include "storage/trie/polkadot_trie/polkadot_trie_cursor_impl.hpp" namespace kagome::storage::trie { @@ -38,6 +39,10 @@ namespace kagome::storage::trie { MOCK_METHOD(std::optional, key, (), (const, override)); MOCK_METHOD(std::optional, value, (), (const, override)); + MOCK_METHOD(std::optional, + valueHash, + (), + (const, override)); }; } // namespace kagome::storage::trie diff --git a/test/mock/core/storage/trie/trie_storage_backend_mock.hpp b/test/mock/core/storage/trie/trie_storage_backend_mock.hpp index c27288b9f7..a8b74a420e 100644 --- a/test/mock/core/storage/trie/trie_storage_backend_mock.hpp +++ b/test/mock/core/storage/trie/trie_storage_backend_mock.hpp @@ -14,37 +14,9 @@ namespace kagome::storage::trie { class TrieStorageBackendMock : public TrieStorageBackend { public: - MOCK_METHOD(std::unique_ptr, batch, (), (override)); - - MOCK_METHOD((std::unique_ptr>), - cursor, - (), - (override)); - - MOCK_METHOD(outcome::result, - get, - (const BufferView &key), - (const, override)); - - MOCK_METHOD(outcome::result>, - tryGet, - (const BufferView &key), - (const, override)); - - MOCK_METHOD(outcome::result, - contains, - (const BufferView &key), - (const, override)); - - MOCK_METHOD(outcome::result, - put, - (const BufferView &key, BufferOrView &&value), - (override)); - - MOCK_METHOD(outcome::result, - remove, - (const BufferView &key), - (override)); + MOCK_METHOD(BufferStorage &, nodes, (), (override)); + MOCK_METHOD(BufferStorage &, values, (), (override)); + MOCK_METHOD(std::unique_ptr, batch, (), (override)); }; } // namespace kagome::storage::trie diff --git a/test/mock/core/storage/write_batch_mock.hpp b/test/mock/core/storage/write_batch_mock.hpp index ba844d2aed..eb3b4d1e16 100644 --- a/test/mock/core/storage/write_batch_mock.hpp +++ b/test/mock/core/storage/write_batch_mock.hpp @@ -29,4 +29,25 @@ namespace kagome::storage::face { MOCK_METHOD1_T(remove, outcome::result(const View &key)); }; + template + class SpacedBatchMock : public SpacedBatch { + public: + MOCK_METHOD(outcome::result, commit, (), (override)); + + MOCK_METHOD(void, clear, (), (override)); + + MOCK_METHOD3_T(put, + outcome::result(Space space, + const View &key, + const V &value)); + outcome::result put(Space space, + const View &key, + OwnedOrView &&value) override { + return put(space, key, value.mut()); + } + + MOCK_METHOD2_T(remove, + outcome::result(Space space, const View &key)); + }; + } // namespace kagome::storage::face diff --git a/test/testutil/storage/CMakeLists.txt b/test/testutil/storage/CMakeLists.txt index 0c89e8b758..cd4fe10299 100644 --- a/test/testutil/storage/CMakeLists.txt +++ b/test/testutil/storage/CMakeLists.txt @@ -31,3 +31,10 @@ add_library(std_list_adapter INTERFACE) target_link_libraries(std_list_adapter INTERFACE outcome ) + +add_library(in_memory_storage in_memory/in_memory_storage.cpp) +target_link_libraries(in_memory_storage PUBLIC + outcome + blob + storage + ) diff --git a/test/testutil/storage/base_rocksdb_test.cpp b/test/testutil/storage/base_rocksdb_test.cpp index 497151ea5c..d5c3b38565 100644 --- a/test/testutil/storage/base_rocksdb_test.cpp +++ b/test/testutil/storage/base_rocksdb_test.cpp @@ -12,7 +12,7 @@ namespace test { rocksdb::Options options; options.create_if_missing = true; - auto r = RocksDB::create(getPathString(), options); + auto r = RocksDB::create(base_path / "rocksdb", options); rocks_ = std::move(r.value()); db_ = rocks_->getSpace(kagome::storage::Space::kDefault); ASSERT_TRUE(rocks_) << "BaseRocksDB_Test: db is nullptr"; diff --git a/test/testutil/storage/base_rocksdb_test.hpp b/test/testutil/storage/base_rocksdb_test.hpp index 88fe98740f..ddcfe9e055 100644 --- a/test/testutil/storage/base_rocksdb_test.hpp +++ b/test/testutil/storage/base_rocksdb_test.hpp @@ -6,6 +6,7 @@ #pragma once +#include "storage/buffer_map_types.hpp" #include "testutil/storage/base_fs_test.hpp" #include "storage/rocksdb/rocksdb.hpp" @@ -24,7 +25,7 @@ namespace test { void TearDown() override; std::shared_ptr rocks_; - std::shared_ptr db_; + std::shared_ptr db_; }; } // namespace test diff --git a/core/storage/in_memory/cursor.hpp b/test/testutil/storage/in_memory/cursor.hpp similarity index 96% rename from core/storage/in_memory/cursor.hpp rename to test/testutil/storage/in_memory/cursor.hpp index 52b7b79ae9..6f35979cd2 100644 --- a/core/storage/in_memory/cursor.hpp +++ b/test/testutil/storage/in_memory/cursor.hpp @@ -7,7 +7,7 @@ #pragma once #include "common/buffer.hpp" -#include "storage/in_memory/in_memory_storage.hpp" +#include "testutil/storage/in_memory/in_memory_storage.hpp" namespace kagome::storage { class InMemoryCursor : public BufferStorageCursor { diff --git a/test/testutil/storage/in_memory/in_memory_batch.hpp b/test/testutil/storage/in_memory/in_memory_batch.hpp new file mode 100644 index 0000000000..d9a02c8da2 --- /dev/null +++ b/test/testutil/storage/in_memory/in_memory_batch.hpp @@ -0,0 +1,84 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "common/buffer.hpp" +#include "storage/buffer_map_types.hpp" +#include "testutil/storage/in_memory/in_memory_spaced_storage.hpp" +#include "testutil/storage/in_memory/in_memory_storage.hpp" + +namespace kagome::storage { + using kagome::common::Buffer; + + class InMemoryBatch : public BufferBatch { + public: + explicit InMemoryBatch(InMemoryStorage &db) : db{db} {} + + outcome::result put(const BufferView &key, + BufferOrView &&value) override { + entries[key.toHex()] = std::move(value).intoBuffer(); + return outcome::success(); + } + + outcome::result remove(const BufferView &key) override { + entries.erase(key.toHex()); + return outcome::success(); + } + + outcome::result commit() override { + for (auto &entry : entries) { + OUTCOME_TRY(db.put(Buffer::fromHex(entry.first).value(), + BufferView{entry.second})); + } + return outcome::success(); + } + + void clear() override { + entries.clear(); + } + + private: + std::map entries; + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + InMemoryStorage &db; + }; + + class InMemorySpacedBatch : public BufferSpacedBatch { + public: + explicit InMemorySpacedBatch(SpacedStorage &db) : db{db} {} + + outcome::result put(Space space, + const BufferView &key, + BufferOrView &&value) override { + entries[std::make_pair(space, key.toHex())] = + std::move(value).intoBuffer(); + return outcome::success(); + } + + outcome::result remove(Space space, const BufferView &key) override { + entries.erase(std::make_pair(space, key.toHex())); + return outcome::success(); + } + + outcome::result commit() override { + for (auto &[key, entry] : entries) { + OUTCOME_TRY(db.getSpace(key.first)->put( + Buffer::fromHex(key.second).value(), BufferView{entry})); + } + return outcome::success(); + } + + void clear() override { + entries.clear(); + } + + private: + std::map, Buffer> entries; + // NOLINTNEXTLINE(cppcoreguidelines-avoid-const-or-ref-data-members) + SpacedStorage &db; + }; +} // namespace kagome::storage diff --git a/core/storage/in_memory/in_memory_spaced_storage.hpp b/test/testutil/storage/in_memory/in_memory_spaced_storage.hpp similarity index 76% rename from core/storage/in_memory/in_memory_spaced_storage.hpp rename to test/testutil/storage/in_memory/in_memory_spaced_storage.hpp index 0a257c6866..d5c0c43cba 100644 --- a/core/storage/in_memory/in_memory_spaced_storage.hpp +++ b/test/testutil/storage/in_memory/in_memory_spaced_storage.hpp @@ -14,8 +14,8 @@ #include "in_memory_storage.hpp" #include "outcome/outcome.hpp" #include "storage/buffer_map_types.hpp" -#include "storage/in_memory/in_memory_storage.hpp" #include "storage/spaced_storage.hpp" +#include "testutil/storage/in_memory/in_memory_storage.hpp" namespace kagome::storage { @@ -24,9 +24,9 @@ namespace kagome::storage { * Mostly needed to have an in-memory trie in tests to avoid integration with * an actual persistent database */ - class InMemorySpacedStorage : public storage::SpacedStorage { + class InMemorySpacedStorage final : public SpacedStorage { public: - std::shared_ptr getSpace(Space space) override { + std::shared_ptr getSpace(Space space) override { auto it = spaces.find(space); if (it != spaces.end()) { return it->second; @@ -35,6 +35,8 @@ namespace kagome::storage { .first->second; } + std::unique_ptr createBatch() override; + private: std::map> spaces; }; diff --git a/core/storage/in_memory/in_memory_storage.cpp b/test/testutil/storage/in_memory/in_memory_storage.cpp similarity index 86% rename from core/storage/in_memory/in_memory_storage.cpp rename to test/testutil/storage/in_memory/in_memory_storage.cpp index e1862a3a65..3dd0d6bd1e 100644 --- a/core/storage/in_memory/in_memory_storage.cpp +++ b/test/testutil/storage/in_memory/in_memory_storage.cpp @@ -4,11 +4,11 @@ * SPDX-License-Identifier: Apache-2.0 */ -#include "storage/in_memory/in_memory_storage.hpp" +#include "testutil/storage/in_memory/in_memory_storage.hpp" #include "storage/database_error.hpp" -#include "storage/in_memory/cursor.hpp" -#include "storage/in_memory/in_memory_batch.hpp" +#include "testutil/storage/in_memory/cursor.hpp" +#include "testutil/storage/in_memory/in_memory_batch.hpp" using kagome::common::Buffer; @@ -69,4 +69,8 @@ namespace kagome::storage { std::optional InMemoryStorage::byteSizeHint() const { return size_; } + + std::unique_ptr InMemorySpacedStorage::createBatch() { + return std::make_unique(*this); + } } // namespace kagome::storage diff --git a/core/storage/in_memory/in_memory_storage.hpp b/test/testutil/storage/in_memory/in_memory_storage.hpp similarity index 90% rename from core/storage/in_memory/in_memory_storage.hpp rename to test/testutil/storage/in_memory/in_memory_storage.hpp index dc777c1c59..dddec04384 100644 --- a/core/storage/in_memory/in_memory_storage.hpp +++ b/test/testutil/storage/in_memory/in_memory_storage.hpp @@ -11,6 +11,8 @@ #include "common/buffer.hpp" #include "outcome/outcome.hpp" #include "storage/buffer_map_types.hpp" +#include "storage/face/batch_writeable.hpp" +#include "storage/face/write_batch.hpp" namespace kagome::storage { @@ -19,7 +21,7 @@ namespace kagome::storage { * Mostly needed to have an in-memory trie in tests to avoid integration with * an actual persistent database */ - class InMemoryStorage : public storage::BufferStorage { + class InMemoryStorage : public BufferBatchableStorage { public: ~InMemoryStorage() override = default; From f0661f9e1e40e111e50045c41c0b214d00d0c38c Mon Sep 17 00:00:00 2001 From: kamilsa Date: Tue, 3 Dec 2024 14:05:30 +0500 Subject: [PATCH 09/17] Add include (#2301) --- core/storage/rocksdb/rocksdb_spaces.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/core/storage/rocksdb/rocksdb_spaces.cpp b/core/storage/rocksdb/rocksdb_spaces.cpp index 7eabdfc841..bd72b97eb5 100644 --- a/core/storage/rocksdb/rocksdb_spaces.cpp +++ b/core/storage/rocksdb/rocksdb_spaces.cpp @@ -7,6 +7,7 @@ #include "storage/rocksdb/rocksdb_spaces.hpp" #include +#include #include #include From 5418b4e91baffac42ec0532d4536dd8da9d60b54 Mon Sep 17 00:00:00 2001 From: kamilsa Date: Thu, 5 Dec 2024 13:06:52 +0500 Subject: [PATCH 10/17] Fix queue_to_connect_ & UpgradeRestriction type (#2306) * Fix queue_to_connect_ * Fix UpgradeRestriction --- core/network/impl/peer_manager_impl.cpp | 3 ++- core/parachain/types.hpp | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/core/network/impl/peer_manager_impl.cpp b/core/network/impl/peer_manager_impl.cpp index 53cf6f7caf..90757356c8 100644 --- a/core/network/impl/peer_manager_impl.cpp +++ b/core/network/impl/peer_manager_impl.cpp @@ -343,7 +343,8 @@ namespace kagome::network { // Not enough active peers if (countPeers(PeerType::PEER_TYPE_OUT) < app_config_.outPeers()) { if (not queue_to_connect_.empty()) { - for (;;) { + BOOST_ASSERT(queue_to_connect_.size() == peers_in_queue_.size()); + while (not queue_to_connect_.empty() && not peers_in_queue_.empty()) { auto node = peers_in_queue_.extract(queue_to_connect_.front()); auto &peer_id = node.value(); diff --git a/core/parachain/types.hpp b/core/parachain/types.hpp index 572778dda6..b2ea0e21e0 100644 --- a/core/parachain/types.hpp +++ b/core/parachain/types.hpp @@ -221,7 +221,7 @@ namespace kagome::network { } // namespace kagome::network namespace kagome::parachain::fragment { - enum UpgradeRestriction { + enum UpgradeRestriction : uint8_t { /// There is an upgrade restriction and there are no details about its /// specifics nor how long /// it could last. From 370cd2eb43512720e369884f2c362065e590c6c4 Mon Sep 17 00:00:00 2001 From: kamilsa Date: Mon, 9 Dec 2024 16:57:25 +0500 Subject: [PATCH 11/17] Audi store (#2303) * Audi store Co-authored-by: Ruslan Tushov --- core/authority_discovery/CMakeLists.txt | 7 +- core/authority_discovery/query/audi_store.hpp | 71 +++++++++++ .../query/audi_store_impl.cpp | 118 ++++++++++++++++++ .../query/audi_store_impl.hpp | 46 +++++++ .../query/authority_peer_info.hpp | 24 ++++ core/authority_discovery/query/query_impl.cpp | 66 ++++++---- core/authority_discovery/query/query_impl.hpp | 12 +- core/injector/application_injector.cpp | 2 + core/scale/kagome_scale.hpp | 14 +++ core/storage/rocksdb/rocksdb_spaces.cpp | 6 +- core/storage/spaces.hpp | 1 + 11 files changed, 327 insertions(+), 40 deletions(-) create mode 100644 core/authority_discovery/query/audi_store.hpp create mode 100644 core/authority_discovery/query/audi_store_impl.cpp create mode 100644 core/authority_discovery/query/audi_store_impl.hpp create mode 100644 core/authority_discovery/query/authority_peer_info.hpp diff --git a/core/authority_discovery/CMakeLists.txt b/core/authority_discovery/CMakeLists.txt index 23736cfd4c..f4e6477baa 100644 --- a/core/authority_discovery/CMakeLists.txt +++ b/core/authority_discovery/CMakeLists.txt @@ -9,9 +9,12 @@ add_subdirectory(protobuf) add_library(address_publisher publisher/address_publisher.cpp query/query_impl.cpp - ) + query/audi_store_impl.cpp + query/audi_store_impl.hpp +) target_link_libraries(address_publisher authority_discovery_proto logger + scale_libp2p_types sha - ) +) diff --git a/core/authority_discovery/query/audi_store.hpp b/core/authority_discovery/query/audi_store.hpp new file mode 100644 index 0000000000..83ad76fe23 --- /dev/null +++ b/core/authority_discovery/query/audi_store.hpp @@ -0,0 +1,71 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include "authority_discovery/query/authority_peer_info.hpp" +#include "common/buffer.hpp" +#include "primitives/authority_discovery_id.hpp" +#include "storage/face/map_cursor.hpp" + +namespace kagome::authority_discovery { + + /** + * Interface for storing and retrieving authority discovery data + */ + class AudiStore { + public: + virtual ~AudiStore() = default; + + /** + * Store authority discovery data + * @param authority the authority to store + * @param data the data to store + */ + virtual void store(const primitives::AuthorityDiscoveryId &authority, + const AuthorityPeerInfo &data) = 0; + + /** + * Get authority discovery data + * @param authority the authority to get + * @return the data if it exists, otherwise std::nullopt + */ + virtual std::optional get( + const primitives::AuthorityDiscoveryId &authority) const = 0; + + /** + * Remove authority discovery data + * @param authority the authority to remove + */ + virtual outcome::result remove( + const primitives::AuthorityDiscoveryId &authority) = 0; + + /** + * Check if the store contains the authority + * @param authority the authority to check + * @return true if the authority is in the store, false otherwise + */ + virtual bool contains( + const primitives::AuthorityDiscoveryId &authority) const = 0; + + /** + * Apply a function to each entry in the store + */ + virtual void forEach( + std::function f) const = 0; + + /** + * Retain only the entries that satisfy the predicate + */ + virtual void retainIf( + std::function f) = 0; + }; + +} // namespace kagome::authority_discovery diff --git a/core/authority_discovery/query/audi_store_impl.cpp b/core/authority_discovery/query/audi_store_impl.cpp new file mode 100644 index 0000000000..7c48a3b03c --- /dev/null +++ b/core/authority_discovery/query/audi_store_impl.cpp @@ -0,0 +1,118 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#include "authority_discovery/query/audi_store_impl.hpp" + +#include + +namespace kagome::authority_discovery { + + AudiStoreImpl::AudiStoreImpl(std::shared_ptr storage) + : space_{storage->getSpace(storage::Space::kAudiPeers)} { + BOOST_ASSERT(space_ != nullptr); + } + + void AudiStoreImpl::store(const primitives::AuthorityDiscoveryId &authority, + const AuthorityPeerInfo &data) { + auto encoded = scale::encode(data); + + if (not encoded) { + SL_ERROR(log_, "Failed to encode PeerInfo"); + return; + } + + if (auto res = space_->put(authority, common::Buffer(encoded.value())); + not res) { + SL_ERROR( + log_, "Failed to put authority {} error {}", authority, res.error()); + } + } + + std::optional AudiStoreImpl::get( + const primitives::AuthorityDiscoveryId &authority) const { + auto res = space_->tryGet(authority); + + // check if there was any critical error during reading + if (not res) { + SL_CRITICAL(log_, + "Failed to get authority {} due to database error {}", + authority, + res.error()); + return std::nullopt; + } + + // check if the authority was not found + if (not res.value()) { + return std::nullopt; + } + + auto decoded = scale::decode(res.value().value()); + + if (not decoded) { + SL_ERROR(log_, "Failed to decode PeerInfo"); + return std::nullopt; + } + + return decoded.value(); + } + + outcome::result AudiStoreImpl::remove( + const primitives::AuthorityDiscoveryId &authority) { + return space_->remove(authority); + } + + bool AudiStoreImpl::contains( + const primitives::AuthorityDiscoveryId &authority) const { + auto res = space_->tryGet(authority); + return res.has_value() and res.value().has_value(); + } + + void AudiStoreImpl::forEach( + std::function f) const { + auto cursor = space_->cursor(); + std::ignore = cursor->seekFirst(); + while (cursor->isValid()) { + auto key = cursor->key(); + auto value = cursor->value(); + if (not key or not value + or key.value().size() != primitives::AuthorityDiscoveryId::size()) { + std::ignore = cursor->next(); + continue; + } + primitives::AuthorityDiscoveryId authority = + primitives::AuthorityDiscoveryId::fromSpan(key.value().toVector()) + .value(); + if (auto decoded = scale::decode(value.value())) { + f(authority, decoded.value()); + } else { + SL_ERROR(log_, "Failed to decode PeerInfo"); + } + std::ignore = cursor->next(); + } + } + + void AudiStoreImpl::retainIf( + std::function f) { + std::deque to_remove; + forEach([&](const auto &authority, const auto &peer_info) { + if (not f(authority, peer_info)) { + to_remove.push_back(authority); + } + }); + for (const auto &authority : to_remove) { + auto res = remove(authority); + if (not res) { + SL_ERROR(log_, + "Failed to remove authority {} due to db error {}", + authority, + res.error()); + } + } + } + +} // namespace kagome::authority_discovery diff --git a/core/authority_discovery/query/audi_store_impl.hpp b/core/authority_discovery/query/audi_store_impl.hpp new file mode 100644 index 0000000000..381e8e3e35 --- /dev/null +++ b/core/authority_discovery/query/audi_store_impl.hpp @@ -0,0 +1,46 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "authority_discovery/query/audi_store.hpp" +#include "log/logger.hpp" +#include "primitives/authority_discovery_id.hpp" +#include "storage/spaced_storage.hpp" + +namespace kagome::authority_discovery { + + class AudiStoreImpl : public AudiStore { + public: + explicit AudiStoreImpl(std::shared_ptr storage); + + ~AudiStoreImpl() override = default; + + void store(const primitives::AuthorityDiscoveryId &authority, + const AuthorityPeerInfo &data) override; + + std::optional get( + const primitives::AuthorityDiscoveryId &authority) const override; + + outcome::result remove( + const primitives::AuthorityDiscoveryId &authority) override; + + bool contains( + const primitives::AuthorityDiscoveryId &authority) const override; + + void forEach( + std::function f) const override; + + void retainIf(std::function f) override; + + private: + std::shared_ptr space_; + log::Logger log_; + }; + +} // namespace kagome::authority_discovery diff --git a/core/authority_discovery/query/authority_peer_info.hpp b/core/authority_discovery/query/authority_peer_info.hpp new file mode 100644 index 0000000000..056da258ad --- /dev/null +++ b/core/authority_discovery/query/authority_peer_info.hpp @@ -0,0 +1,24 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include "authority_discovery/query/authority_peer_info.hpp" +#include "authority_discovery/timestamp.hpp" +#include "common/buffer.hpp" +#include "scale/libp2p_types.hpp" +#include "scale/tie.hpp" + +namespace kagome::authority_discovery { + struct AuthorityPeerInfo { + SCALE_TIE(3); + + common::Buffer raw{}; + std::optional time{}; + scale::PeerInfoSerializable peer{}; + }; + +} // namespace kagome::authority_discovery diff --git a/core/authority_discovery/query/query_impl.cpp b/core/authority_discovery/query/query_impl.cpp index dba9020b6d..50b9703c78 100644 --- a/core/authority_discovery/query/query_impl.cpp +++ b/core/authority_discovery/query/query_impl.cpp @@ -41,6 +41,7 @@ namespace kagome::authority_discovery { std::shared_ptr authority_discovery_api, LazySPtr validation_protocol, std::shared_ptr key_store, + std::shared_ptr audi_store, std::shared_ptr sr_crypto_provider, std::shared_ptr libp2p_crypto_provider, std::shared_ptr key_marshaller, @@ -51,6 +52,7 @@ namespace kagome::authority_discovery { authority_discovery_api_{std::move(authority_discovery_api)}, validation_protocol_{std::move(validation_protocol)}, key_store_{std::move(key_store)}, + audi_store_(std::move(audi_store)), sr_crypto_provider_{std::move(sr_crypto_provider)}, libp2p_crypto_provider_{std::move(libp2p_crypto_provider)}, key_marshaller_{std::move(key_marshaller)}, @@ -68,6 +70,7 @@ namespace kagome::authority_discovery { log_{log::createLogger("AuthorityDiscoveryQuery", "authority_discovery")} { app_state_manager->takeControl(*this); + BOOST_ASSERT(audi_store_ != nullptr); } bool QueryImpl::start() { @@ -83,11 +86,18 @@ namespace kagome::authority_discovery { std::optional QueryImpl::get( const primitives::AuthorityDiscoveryId &authority) const { std::unique_lock lock{mutex_}; - auto it = auth_to_peer_cache_.find(authority); - if (it != auth_to_peer_cache_.end()) { - return it->second.peer; + auto authority_opt = audi_store_->get(authority); + if (authority_opt == std::nullopt) { + SL_TRACE(log_, + "No authority peer found in storage {}", + common::hex_lower(authority)); + return std::nullopt; } - return std::nullopt; + SL_TRACE(log_, + "Authority id {} {} addresses found in storage", + common::hex_lower(authority), + authority_opt.value().peer.addresses.size()); + return authority_opt.value().peer; } std::optional QueryImpl::get( @@ -125,9 +135,9 @@ namespace kagome::authority_discovery { lock.unlock(); return kademlia_validator_.select(key, values); } - auto it = auth_to_peer_cache_.find(*id); - if (it != auth_to_peer_cache_.end()) { - auto it_value = std::ranges::find(values, it->second.raw); + auto authority = audi_store_->get(*id); + if (authority != std::nullopt) { + auto it_value = std::ranges::find(values, authority.value().raw); if (it_value != values.end()) { return it_value - values.begin(); } @@ -155,15 +165,15 @@ namespace kagome::authority_discovery { retain_if(authorities, [&](const primitives::AuthorityDiscoveryId &id) { return not has(local_keys, id); }); - for (auto it = auth_to_peer_cache_.begin(); - it != auth_to_peer_cache_.end();) { - if (has(authorities, it->first)) { - ++it; - } else { - it = auth_to_peer_cache_.erase(it); - validation_protocol_.get()->reserve(it->second.peer.id, false); + // remove outdated authorities + audi_store_->retainIf([&](const primitives::AuthorityDiscoveryId &id, + const AuthorityPeerInfo &info) { + if (has(authorities, id)) { + return true; } - } + validation_protocol_.get()->reserve(info.peer.id, false); + return false; + }); for (auto it = peer_to_auth_cache_.begin(); it != peer_to_auth_cache_.end();) { if (has(authorities, it->second)) { @@ -181,7 +191,7 @@ namespace kagome::authority_discovery { // `queue_` is a stack, so push known first for (auto &known : {true, false}) { for (auto &id : authorities) { - if (auth_to_peer_cache_.contains(id) == known) { + if (audi_store_->contains(id) == known) { queue_.emplace_back(id); } } @@ -238,9 +248,8 @@ namespace kagome::authority_discovery { common::hex_lower(authority), _res.has_value() ? "ok" : "error: " + _res.error().message()); OUTCOME_TRY(signed_record_pb, _res); - auto it = auth_to_peer_cache_.find(authority); - if (it != auth_to_peer_cache_.end() - and signed_record_pb == it->second.raw) { + auto it = audi_store_->get(authority); + if (it != std::nullopt and signed_record_pb == it->raw) { return outcome::success(); } ::authority_discovery_v3::SignedAuthorityRecord signed_record; @@ -281,12 +290,13 @@ namespace kagome::authority_discovery { scale::decode( qtils::str2byte(record.creation_time().timestamp()))); time = *tmp; - if (it != auth_to_peer_cache_.end() and time <= it->second.time) { + if (it and it->time and time <= it->time->number) { SL_TRACE(log_, "lookup: outdated record for authority {}", authority); return outcome::success(); } } - libp2p::peer::PeerInfo peer{.id = std::move(peer_id)}; + scale::PeerInfoSerializable peer; + peer.id = std::move(peer_id); auto peer_id_str = peer.id.toBase58(); SL_TRACE(log_, "lookup: adding {} addresses for authority {}", @@ -330,12 +340,14 @@ namespace kagome::authority_discovery { peer.id, peer.addresses, libp2p::peer::ttl::kDay); peer_to_auth_cache_.insert_or_assign(peer.id, authority); - auth_to_peer_cache_.insert_or_assign(authority, - Authority{ - .raw = std::move(signed_record_pb), - .time = time, - .peer = peer, - }); + audi_store_->store( + authority, + AuthorityPeerInfo{ + .raw = std::move(signed_record_pb), + .time = time.has_value() ? std::make_optional(*time) + : std::nullopt, + .peer = peer, + }); validation_protocol_.get()->reserve(peer.id, true); diff --git a/core/authority_discovery/query/query_impl.hpp b/core/authority_discovery/query/query_impl.hpp index 6a135ebbdb..896d5b4e16 100644 --- a/core/authority_discovery/query/query_impl.hpp +++ b/core/authority_discovery/query/query_impl.hpp @@ -10,7 +10,7 @@ #include "application/app_state_manager.hpp" #include "authority_discovery/interval.hpp" -#include "authority_discovery/timestamp.hpp" +#include "authority_discovery/query/audi_store.hpp" #include "blockchain/block_tree.hpp" #include "crypto/key_store.hpp" #include "crypto/sr25519_provider.hpp" @@ -49,6 +49,7 @@ namespace kagome::authority_discovery { std::shared_ptr authority_discovery_api, LazySPtr validation_protocol, std::shared_ptr key_store, + std::shared_ptr audi_store, std::shared_ptr sr_crypto_provider, std::shared_ptr libp2p_crypto_provider, std::shared_ptr @@ -76,12 +77,6 @@ namespace kagome::authority_discovery { outcome::result update(); private: - struct Authority { - Buffer raw; - std::optional time; - libp2p::peer::PeerInfo peer; - }; - std::optional hashToAuth( BufferView key) const; void pop(); @@ -92,6 +87,7 @@ namespace kagome::authority_discovery { std::shared_ptr authority_discovery_api_; LazySPtr validation_protocol_; std::shared_ptr key_store_; + std::shared_ptr audi_store_; std::shared_ptr sr_crypto_provider_; std::shared_ptr libp2p_crypto_provider_; std::shared_ptr key_marshaller_; @@ -104,8 +100,6 @@ namespace kagome::authority_discovery { std::default_random_engine random_; libp2p::protocol::kademlia::ValidatorDefault kademlia_validator_; std::unordered_map hash_to_auth_; - std::unordered_map - auth_to_peer_cache_; std::unordered_map peer_to_auth_cache_; std::vector queue_; diff --git a/core/injector/application_injector.cpp b/core/injector/application_injector.cpp index 23c49e26cd..b5c2ba0b57 100644 --- a/core/injector/application_injector.cpp +++ b/core/injector/application_injector.cpp @@ -51,6 +51,7 @@ #include "application/modes/print_chain_info_mode.hpp" #include "application/modes/recovery_mode.hpp" #include "authority_discovery/publisher/address_publisher.hpp" +#include "authority_discovery/query/audi_store_impl.hpp" #include "authority_discovery/query/query_impl.hpp" #include "authorship/impl/block_builder_factory_impl.hpp" #include "authorship/impl/block_builder_impl.hpp" @@ -869,6 +870,7 @@ namespace { di::bind.template to(), di::bind.template to(), di::bind.template to(), + di::bind.template to(), di::bind.template to(), di::bind.template to()[boost::di::override], di::bind.template to(), diff --git a/core/scale/kagome_scale.hpp b/core/scale/kagome_scale.hpp index d5adba070e..fde89b7b95 100644 --- a/core/scale/kagome_scale.hpp +++ b/core/scale/kagome_scale.hpp @@ -22,6 +22,8 @@ #include "scale/encode_append.hpp" #include "scale/libp2p_types.hpp" +#include + namespace kagome::scale { using CompactInteger = ::scale::CompactInteger; using BitVec = ::scale::BitVec; @@ -90,6 +92,10 @@ namespace kagome::scale { constexpr void encode(const F &func, const consensus::babe::BabeBlockHeader &bh); + template + constexpr void encode(const F &func, + const authority_discovery::AuthorityPeerInfo &c); + } // namespace kagome::scale #include "scale/encoder/primitives.hpp" @@ -195,6 +201,14 @@ namespace kagome::scale { encode(func, c.value); } + template + constexpr void encode(const F &func, + const authority_discovery::AuthorityPeerInfo &c) { + encode(func, c.raw); + encode(func, c.time); + encode(func, c.peer); + } + } // namespace kagome::scale #endif // KAGOME_KAGOME_SCALE_HPP diff --git a/core/storage/rocksdb/rocksdb_spaces.cpp b/core/storage/rocksdb/rocksdb_spaces.cpp index bd72b97eb5..36a33cdba1 100644 --- a/core/storage/rocksdb/rocksdb_spaces.cpp +++ b/core/storage/rocksdb/rocksdb_spaces.cpp @@ -6,8 +6,8 @@ #include "storage/rocksdb/rocksdb_spaces.hpp" -#include #include +#include #include #include @@ -22,7 +22,9 @@ namespace kagome::storage { "trie_value", "dispute_data", "beefy_justification", - "avaliability_storage"}; + "avaliability_storage", + "audi_peers", + }; std::string spaceName(Space space) { static_assert(kSpaceNames.size() == Space::kTotal - 1); diff --git a/core/storage/spaces.hpp b/core/storage/spaces.hpp index 3f4ef67dd3..a18f8f81ca 100644 --- a/core/storage/spaces.hpp +++ b/core/storage/spaces.hpp @@ -24,6 +24,7 @@ namespace kagome::storage { kDisputeData, kBeefyJustification, kAvaliabilityStorage, + kAudiPeers, kTotal }; From 97ef53f3089f0502699f5a0bd608c3c22dd64701 Mon Sep 17 00:00:00 2001 From: Boris <31869190+ErakhtinB@users.noreply.github.com> Date: Tue, 10 Dec 2024 19:26:41 +0500 Subject: [PATCH 12/17] 2299 feature request kagome node key generate util (#2308) --- core/application/CMakeLists.txt | 8 +++ .../impl/app_configuration_impl.cpp | 2 +- core/application/modes/key.hpp | 55 +++++++++++++++++++ core/application/modes/key_main.cpp | 41 ++++++++++++++ core/injector/CMakeLists.txt | 1 + core/injector/application_injector.cpp | 5 ++ core/injector/application_injector.hpp | 6 ++ node/CMakeLists.txt | 1 + node/main.cpp | 5 ++ 9 files changed, 123 insertions(+), 1 deletion(-) create mode 100644 core/application/modes/key.hpp create mode 100644 core/application/modes/key_main.cpp diff --git a/core/application/CMakeLists.txt b/core/application/CMakeLists.txt index 85a3012494..772f55561c 100644 --- a/core/application/CMakeLists.txt +++ b/core/application/CMakeLists.txt @@ -83,6 +83,14 @@ target_link_libraries(recovery_mode logger ) +add_library(kagome_key + modes/key_main.cpp +) + +target_link_libraries(kagome_key + application_injector +) + add_library(kagome_application impl/kagome_application_impl.cpp ) diff --git a/core/application/impl/app_configuration_impl.cpp b/core/application/impl/app_configuration_impl.cpp index dd5deb2c0a..7e34615377 100644 --- a/core/application/impl/app_configuration_impl.cpp +++ b/core/application/impl/app_configuration_impl.cpp @@ -913,7 +913,7 @@ namespace kagome::application { if (vm.count("help") > 0) { std::cout - << "Available subcommands: storage-explorer db-editor benchmark\n"; + << "Available subcommands: storage-explorer db-editor benchmark key\n"; std::cout << desc << '\n'; return false; } diff --git a/core/application/modes/key.hpp b/core/application/modes/key.hpp new file mode 100644 index 0000000000..aab200c665 --- /dev/null +++ b/core/application/modes/key.hpp @@ -0,0 +1,55 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include + +#include +#include +#include "crypto/ed25519_provider.hpp" +#include "crypto/key_store.hpp" +#include "crypto/random_generator/boost_generator.hpp" + +namespace kagome::key { + + class Key { + public: + Key(std::shared_ptr ed_crypto_provider, + std::shared_ptr + key_marshaller) + : ed_crypto_provider_{std::move(ed_crypto_provider)}, + key_marshaller_{std::move(key_marshaller)} { + BOOST_ASSERT(ed_crypto_provider_ != nullptr); + BOOST_ASSERT(key_marshaller_ != nullptr); + } + + outcome::result run() { + auto random_generator = std::make_unique(); + OUTCOME_TRY( + seed, + crypto::Ed25519Seed::from(crypto::SecureCleanGuard{ + random_generator->randomBytes(crypto::Ed25519Seed::size())})); + OUTCOME_TRY(keypair, ed_crypto_provider_->generateKeypair(seed, {})); + const auto libp2p_key = crypto::ed25519KeyToLibp2pKeypair(keypair); + const libp2p::crypto::ProtobufKey protobuf_key{ + key_marshaller_->marshal(libp2p_key.publicKey).value()}; + auto peer_id = libp2p::peer::PeerId::fromPublicKey(protobuf_key); + if (not peer_id) { + return peer_id.error(); + } + std::cerr << peer_id.value().toBase58() << "\n"; + std::cout << kagome::common::hex_lower(keypair.secret_key.unsafeBytes()) + << "\n"; + + return outcome::success(); + } + + private: + std::shared_ptr ed_crypto_provider_; + std::shared_ptr key_marshaller_; + }; +} // namespace kagome::key diff --git a/core/application/modes/key_main.cpp b/core/application/modes/key_main.cpp new file mode 100644 index 0000000000..9cd385a550 --- /dev/null +++ b/core/application/modes/key_main.cpp @@ -0,0 +1,41 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ +#include "application/impl/app_configuration_impl.hpp" +#include "injector/application_injector.hpp" +#include "key.hpp" + +namespace kagome { + int key_main(int argc, const char **argv) { + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) + const std::string_view key_command = argv[0]; + if (argc == 2) { + // NOLINTNEXTLINE(cppcoreguidelines-pro-bounds-pointer-arithmetic) + const std::string_view command = argv[1]; + if (command == "--generate-node-key") { + auto injector = std::make_unique( + std::make_shared()); + auto key = injector->injectKey(); + if (auto res = key->run(); not res) { + std::cerr << "Error: " << res.error().message() << "\n"; + return 2; + } + } else if (command == "--help") { + std::cerr << "Usage: " << key_command << " --generate-node-key" + << "\nGenerates a node key and prints the peer ID to stderr " + "and the secret key to stdout.\n"; + } else { + std::cerr << "Unknown command: " << command << "\n"; + std::cerr << "Usage: " << key_command << " --generate-node-key\n"; + return 3; + } + } else { + std::cerr << "Usage: " << key_command << " --generate-node-key\n"; + return 1; + } + return 0; + } + +} // namespace kagome diff --git a/core/injector/CMakeLists.txt b/core/injector/CMakeLists.txt index ee180e700b..924c0134fd 100644 --- a/core/injector/CMakeLists.txt +++ b/core/injector/CMakeLists.txt @@ -45,6 +45,7 @@ target_link_libraries(application_injector grandpa_api host_api_factory kagome_benchmarks + kagome_key log_configurator metadata_api metrics diff --git a/core/injector/application_injector.cpp b/core/injector/application_injector.cpp index b5c2ba0b57..5cd324eb08 100644 --- a/core/injector/application_injector.cpp +++ b/core/injector/application_injector.cpp @@ -50,6 +50,7 @@ #include "application/modes/precompile_wasm.hpp" #include "application/modes/print_chain_info_mode.hpp" #include "application/modes/recovery_mode.hpp" +#include "application/modes/key.hpp" #include "authority_discovery/publisher/address_publisher.hpp" #include "authority_discovery/query/audi_store_impl.hpp" #include "authority_discovery/query/query_impl.hpp" @@ -1119,6 +1120,10 @@ namespace kagome::injector { .template create>(); } + std::shared_ptr KagomeNodeInjector::injectKey() { + return pimpl_->injector_.template create>(); + } + std::shared_ptr KagomeNodeInjector::injectWatchdog() { return pimpl_->injector_.template create>(); } diff --git a/core/injector/application_injector.hpp b/core/injector/application_injector.hpp index ea68b4ebfe..efa9b1fcca 100644 --- a/core/injector/application_injector.hpp +++ b/core/injector/application_injector.hpp @@ -31,6 +31,7 @@ namespace kagome { class PrecompileWasmMode; class RecoveryMode; class BenchmarkMode; + class Key; } // namespace application::mode namespace authority_discovery { @@ -104,6 +105,10 @@ namespace kagome { class TelemetryService; } + namespace key { + class Key; + } + class Watchdog; } // namespace kagome @@ -163,6 +168,7 @@ namespace kagome::injector { injectPrecompileWasmMode(); std::shared_ptr injectRecoveryMode(); std::shared_ptr injectBlockBenchmark(); + std::shared_ptr injectKey(); protected: std::shared_ptr pimpl_; diff --git a/node/CMakeLists.txt b/node/CMakeLists.txt index 1ad9c128a1..1a0bf7c33c 100644 --- a/node/CMakeLists.txt +++ b/node/CMakeLists.txt @@ -32,6 +32,7 @@ target_link_libraries(kagome kagome-db-editor storage_explorer filesystem + kagome_key ) if (BACKWARD) add_backward(kagome) diff --git a/node/main.cpp b/node/main.cpp index 8fe1385b6b..f6492370f9 100644 --- a/node/main.cpp +++ b/node/main.cpp @@ -35,6 +35,7 @@ int db_editor_main(int argc, const char **argv); namespace kagome { int benchmark_main(int argc, const char **argv); + int key_main(int argc, const char **argv); } namespace { @@ -184,6 +185,10 @@ int main(int argc, const char **argv, const char **env) { exit_code = kagome::benchmark_main(argc - 1, argv + 1); } + else if (name == "key") { + exit_code = kagome::key_main(argc - 1, argv + 1); + } + else if (name.substr(0, 1) == "-") { // The first argument isn't subcommand, run as node exit_code = run_node(argc, argv); From d988d348c887d03172de11dd8f233b4ae8b61849 Mon Sep 17 00:00:00 2001 From: Alexander Lednev <57529355+iceseer@users.noreply.github.com> Date: Wed, 11 Dec 2024 13:12:20 +0300 Subject: [PATCH 13/17] Fixup/era points (#2309) * memory fixup Signed-off-by: iceseer * Try fix deadlock * Era points fixup Signed-off-by: iceseer --------- Signed-off-by: iceseer Co-authored-by: kamilsa --- core/blockchain/block_tree.hpp | 1 + core/blockchain/impl/block_tree_impl.cpp | 29 ++++++++++-------- core/blockchain/impl/block_tree_impl.hpp | 1 + core/blockchain/impl/cached_tree.cpp | 30 ++++++++++++++----- core/blockchain/impl/cached_tree.hpp | 3 +- core/network/impl/peer_view.cpp | 13 ++++++-- .../availability/recovery/recovery_impl.cpp | 1 + .../validator/impl/parachain_processor.cpp | 4 +-- test/mock/core/blockchain/block_tree_mock.hpp | 5 ++++ 9 files changed, 63 insertions(+), 24 deletions(-) diff --git a/core/blockchain/block_tree.hpp b/core/blockchain/block_tree.hpp index 8ff72eb36a..70e8d07808 100644 --- a/core/blockchain/block_tree.hpp +++ b/core/blockchain/block_tree.hpp @@ -234,6 +234,7 @@ namespace kagome::blockchain { * @return collection of the leaves */ virtual std::vector getLeaves() const = 0; + virtual std::vector getLeavesInfo() const = 0; /** * Get children of the block with specified hash diff --git a/core/blockchain/impl/block_tree_impl.cpp b/core/blockchain/impl/block_tree_impl.cpp index aa056ba2b5..5b32499069 100644 --- a/core/blockchain/impl/block_tree_impl.cpp +++ b/core/blockchain/impl/block_tree_impl.cpp @@ -425,18 +425,18 @@ namespace kagome::blockchain { std::shared_ptr state_pruner, common::MainThreadPool &main_thread_pool) : block_tree_data_{BlockTreeData{ - .header_repo_ = std::move(header_repo), - .storage_ = std::move(storage), - .state_pruner_ = std::move(state_pruner), - .tree_ = std::make_unique(finalized), - .extrinsic_observer_ = std::move(extrinsic_observer), - .hasher_ = std::move(hasher), - .extrinsic_event_key_repo_ = std::move(extrinsic_event_key_repo), - .justification_storage_policy_ = - std::move(justification_storage_policy), - .genesis_block_hash_ = {}, - .blocks_pruning_ = {app_config.blocksPruning(), finalized.number}, - }}, + .header_repo_ = std::move(header_repo), + .storage_ = std::move(storage), + .state_pruner_ = std::move(state_pruner), + .tree_ = std::make_unique(finalized), + .extrinsic_observer_ = std::move(extrinsic_observer), + .hasher_ = std::move(hasher), + .extrinsic_event_key_repo_ = std::move(extrinsic_event_key_repo), + .justification_storage_policy_ = + std::move(justification_storage_policy), + .genesis_block_hash_ = {}, + .blocks_pruning_ = {app_config.blocksPruning(), finalized.number}, + }}, chain_events_engine_{std::move(chain_events_engine)}, main_pool_handler_{main_thread_pool.handlerStarted()}, extrinsic_events_engine_{std::move(extrinsic_events_engine)} { @@ -1252,6 +1252,11 @@ namespace kagome::blockchain { [&](const BlockTreeData &p) { return getLeavesNoLock(p); }); } + std::vector BlockTreeImpl::getLeavesInfo() const { + return block_tree_data_.sharedAccess( + [&](const BlockTreeData &p) { return p.tree_->leafInfo(); }); + } + BlockTreeImpl::BlockHashVecRes BlockTreeImpl::getChildren( const primitives::BlockHash &block) const { return block_tree_data_.sharedAccess([&](const BlockTreeData &p) diff --git a/core/blockchain/impl/block_tree_impl.hpp b/core/blockchain/impl/block_tree_impl.hpp index c2e2ee74a5..c7e80960bb 100644 --- a/core/blockchain/impl/block_tree_impl.hpp +++ b/core/blockchain/impl/block_tree_impl.hpp @@ -146,6 +146,7 @@ namespace kagome::blockchain { const primitives::BlockHash &target_hash) const override; std::vector getLeaves() const override; + std::vector getLeavesInfo() const override; BlockHashVecRes getChildren( const primitives::BlockHash &block) const override; diff --git a/core/blockchain/impl/cached_tree.cpp b/core/blockchain/impl/cached_tree.cpp index c5703b9688..0332b8f24e 100644 --- a/core/blockchain/impl/cached_tree.cpp +++ b/core/blockchain/impl/cached_tree.cpp @@ -98,7 +98,7 @@ namespace kagome::blockchain { void CachedTree::forceRefreshBest() { std::set, Cmp> candidates; for (auto &leaf : leaves_) { - auto node = find(leaf); + auto node = find(leaf.first); BOOST_ASSERT(node); candidates.emplace(std::move(node)); } @@ -125,8 +125,9 @@ namespace kagome::blockchain { CachedTree::CachedTree(const primitives::BlockInfo &root) : root_{std::make_shared(root)}, best_{root_}, - nodes_{{root.hash, root_}}, - leaves_{root.hash} {} + nodes_{{root.hash, root_}} { + leaves_.emplace(root.hash, root.number); + } primitives::BlockInfo CachedTree::finalized() const { return root_->info; @@ -140,8 +141,23 @@ namespace kagome::blockchain { return leaves_.size(); } + std::vector CachedTree::leafInfo() const { + std::vector output; + output.reserve(leaves_.size()); + std::ranges::transform( + leaves_, std::back_inserter(output), [](const auto &v) { + return primitives::BlockInfo(v.first, v.second); + }); + return output; + } + std::vector CachedTree::leafHashes() const { - return {leaves_.begin(), leaves_.end()}; + std::vector output; + output.reserve(leaves_.size()); + std::ranges::transform(leaves_, + std::back_inserter(output), + [](const auto &v) { return v.first; }); + return output; } bool CachedTree::isLeaf(const primitives::BlockHash &hash) const { @@ -151,7 +167,7 @@ namespace kagome::blockchain { primitives::BlockInfo CachedTree::bestWith( const std::shared_ptr &required) const { std::set, Cmp> candidates; - for (auto &leaf : leaves_) { + for (auto &[leaf, _] : leaves_) { auto node = find(leaf); BOOST_ASSERT(node); candidates.emplace(std::move(node)); @@ -197,7 +213,7 @@ namespace kagome::blockchain { parent->children.emplace_back(new_node); nodes_.emplace(new_node->info.hash, new_node); leaves_.erase(parent->info.hash); - leaves_.emplace(new_node->info.hash); + leaves_.emplace(new_node->info.hash, new_node->info.number); if (not new_node->reverted and new_node->weight() > best_->weight()) { auto old_best = best_; best_ = new_node; @@ -279,7 +295,7 @@ namespace kagome::blockchain { changes.prune.emplace_back(node->info); parent->children.erase(child_it); if (parent->children.empty()) { - leaves_.emplace(parent->info.hash); + leaves_.emplace(parent->info.hash, parent->info.number); } leaves_.erase(leaf_it); if (node == best_) { diff --git a/core/blockchain/impl/cached_tree.hpp b/core/blockchain/impl/cached_tree.hpp index a940884ce8..1ae2c40a52 100644 --- a/core/blockchain/impl/cached_tree.hpp +++ b/core/blockchain/impl/cached_tree.hpp @@ -78,6 +78,7 @@ namespace kagome::blockchain { primitives::BlockInfo best() const; size_t leafCount() const; std::vector leafHashes() const; + std::vector leafInfo() const; bool isLeaf(const primitives::BlockHash &hash) const; primitives::BlockInfo bestWith( const std::shared_ptr &required) const; @@ -106,6 +107,6 @@ namespace kagome::blockchain { std::shared_ptr root_; std::shared_ptr best_; std::unordered_map> nodes_; - std::unordered_set leaves_; + std::unordered_map leaves_; }; } // namespace kagome::blockchain diff --git a/core/network/impl/peer_view.cpp b/core/network/impl/peer_view.cpp index 0854e3df72..f9a68bb3df 100644 --- a/core/network/impl/peer_view.cpp +++ b/core/network/impl/peer_view.cpp @@ -11,9 +11,18 @@ namespace kagome::network { inline View makeView(const LazySPtr &block_tree) { + auto last_finalized = block_tree.get()->getLastFinalized().number; + + std::vector heads; + for (const auto &bi : block_tree.get()->getLeavesInfo()) { + if (bi.number >= last_finalized) { + heads.emplace_back(bi.hash); + } + } + View view{ - .heads_ = block_tree.get()->getLeaves(), - .finalized_number_ = block_tree.get()->getLastFinalized().number, + .heads_ = std::move(heads), + .finalized_number_ = last_finalized, }; std::ranges::sort(view.heads_); return view; diff --git a/core/parachain/availability/recovery/recovery_impl.cpp b/core/parachain/availability/recovery/recovery_impl.cpp index 3f8004a43b..bf5158b198 100644 --- a/core/parachain/availability/recovery/recovery_impl.cpp +++ b/core/parachain/availability/recovery/recovery_impl.cpp @@ -260,6 +260,7 @@ namespace kagome::parachain { candidate_hash, validator_index, peer->id); + lock.unlock(); send_fetch_available_data_request( peer->id, candidate_hash, diff --git a/core/parachain/validator/impl/parachain_processor.cpp b/core/parachain/validator/impl/parachain_processor.cpp index 9bc9a49d54..66fe599a92 100644 --- a/core/parachain/validator/impl/parachain_processor.cpp +++ b/core/parachain/validator/impl/parachain_processor.cpp @@ -722,12 +722,12 @@ namespace kagome::parachain { auto rps_result = construct_per_relay_parent_state(maybe_new, mode_); if (rps_result.has_value()) { our_current_state_.state_by_relay_parent.insert_or_assign( - relay_parent, std::move(rps_result.value())); + maybe_new, std::move(rps_result.value())); } else if (rps_result.error() != Error::KEY_NOT_PRESENT) { SL_TRACE( logger_, "Relay parent state was not created. (relay parent={}, error={})", - relay_parent, + maybe_new, rps_result.error()); } } diff --git a/test/mock/core/blockchain/block_tree_mock.hpp b/test/mock/core/blockchain/block_tree_mock.hpp index 19a4b68c69..726cbd7a02 100644 --- a/test/mock/core/blockchain/block_tree_mock.hpp +++ b/test/mock/core/blockchain/block_tree_mock.hpp @@ -113,6 +113,11 @@ namespace kagome::blockchain { MOCK_METHOD(primitives::BlockInfo, bestBlock, (), (const, override)); + MOCK_METHOD(std::vector, + getLeavesInfo, + (), + (const, override)); + MOCK_METHOD(std::vector, getLeaves, (), From 31f5f80f373586f37b6a05f129538d673fc2a64d Mon Sep 17 00:00:00 2001 From: kamilsa Date: Wed, 11 Dec 2024 20:28:41 +0500 Subject: [PATCH 14/17] Docker compose (#2310) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * new docker-compose with grafana and prometheus This reverts commit ab89a95a00c3363f1ccb1c3bd8490da780e99764. --------- Co-authored-by: Кирилл Азовцев --- docker-compose.yml | 26 - .../docker-compose/kagome/docker-compose.yaml | 103 + .../Kagome _ Node-1733089320320.json | 3966 +++++++++++++++++ .../grafana/provisioning/dashboards.yaml | 6 + .../grafana/provisioning/datasource.yaml | 9 + .../docker-compose/kagome/logcfg.yaml | 129 + .../kagome/prometheus/config/prometheus.yaml | 26 + scripts/.env | 3 + scripts/build.sh | 12 +- scripts/init.sh | 3 - 10 files changed, 4252 insertions(+), 31 deletions(-) delete mode 100644 docker-compose.yml create mode 100644 housekeeping/docker-compose/kagome/docker-compose.yaml create mode 100644 housekeeping/docker-compose/kagome/grafana/dashboards/Kagome _ Node-1733089320320.json create mode 100644 housekeeping/docker-compose/kagome/grafana/provisioning/dashboards.yaml create mode 100644 housekeeping/docker-compose/kagome/grafana/provisioning/datasource.yaml create mode 100644 housekeeping/docker-compose/kagome/logcfg.yaml create mode 100644 housekeeping/docker-compose/kagome/prometheus/config/prometheus.yaml mode change 100644 => 100755 scripts/build.sh diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 3507869a40..0000000000 --- a/docker-compose.yml +++ /dev/null @@ -1,26 +0,0 @@ -# -# Copyright Quadrivium LLC -# All Rights Reserved -# SPDX-License-Identifier: Apache-2.0 -# - -version: '3' - -# usage: -# - docker-compose up -d -# - docker-compose exec dev bash - -services: - dev: - image: qdrvm/kagome-dev:minideb - tty: true - stdin_open: true - working_dir: /app - cap_add: - - SYS_PTRACE - volumes: - - .:/app:rw - environment: - - GITHUB_HUNTER_USERNAME=$GITHUB_HUNTER_USERNAME - - GITHUB_HUNTER_TOKEN=$GITHUB_HUNTER_TOKEN - command: bash diff --git a/housekeeping/docker-compose/kagome/docker-compose.yaml b/housekeeping/docker-compose/kagome/docker-compose.yaml new file mode 100644 index 0000000000..ff8e8695f1 --- /dev/null +++ b/housekeeping/docker-compose/kagome/docker-compose.yaml @@ -0,0 +1,103 @@ +# +# Copyright Quadrivium LLC +# All Rights Reserved +# SPDX-License-Identifier: Apache-2.0 +# + +### Runs a temporary Polkadot container to generate node keys: +# docker run --rm --platform='linux/amd64' \ +# parity/polkadot:latest key generate-node-key 2>&1 | \ +# awk 'NR==1 {print "public="$0} NR==2 {print "private="$0}' > \ +# ./kagome-node/node-key + +### Set up session keys: +# curl -H "Content-Type: application/json" -d '{"id":1, "jsonrpc":"2.0", "method": "author_rotateKeys", "params":[]}' 127.0.0.1:9944 + +services: + kagome-node: + image: qdrvm/kagome:master + container_name: kagome-node + healthcheck: + test: | + CMD-SHELL curl -Ss -H 'Content-Type: application/json' \ + -d '{"id":1, "jsonrpc":"2.0", "method": "system_health", "params":[]}' \ + 127.0.0.1:9944 | grep 'result' + interval: 10s + timeout: 5s + retries: 3 + start_period: 300s + command: + - "kagome" + - "--chain=kusama" +# - "--sync=Warp" # remove after first sync + - "--base-path=/opt/kagome/chain-data" + - "--rpc-port=9944" + - "--logcfg=/opt/kagome/logcfg.yaml" + - "--name=kagome-qdrvm-compose" + - "--listen-addr=/ip4/0.0.0.0/tcp/30363" # ensure 30363 port is publicly accessible if running validator +# - "--public-addr=/ip4/INSERT_ACTUAL_IP_HERE/tcp/30363" + - "--telemetry-url=wss://telemetry.polkadot.io/submit/ 1" + - "--validator" +# - "--node-key=INSERT_PRIVATE_NODE_KEY_HERE" + - "--insecure-validator-i-know-what-i-do" + - "--prometheus-port=9615" + - "--prometheus-external" + dns: + - 8.8.8.8 + - 1.1.1.1 + volumes: + - ./kagome-node/chain-data:/opt/kagome/chain-data + - ./kagome-node/tmp:/tmp/kagome + - ./logcfg.yaml:/opt/kagome/logcfg.yaml:ro + ports: + - "30363:30363" + - "127.0.0.1:9944:9944" + - "9615:9615" + networks: + - kagome-net + restart: "always" + platform: linux/amd64 + stdin_open: true + tty: true + ulimits: + nofile: + soft: 65536 + hard: 65536 + + prometheus: + image: prom/prometheus:latest + container_name: prometheus + command: + - '--config.file=/etc/prometheus/prometheus.yaml' + ports: + - 9090:9090 + restart: unless-stopped + volumes: + - ./prometheus/config:/etc/prometheus + - ./prometheus/data:/prometheus + networks: + - kagome-net + + grafana: + image: grafana/grafana:latest + container_name: grafana + ports: + - 3000:3000 + restart: unless-stopped + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=grafana + - GF_AUTH_ANONYMOUS_ENABLED=true + - GF_AUTH_ANONYMOUS_ORG_ROLE=Admin + volumes: + - ./grafana/provisioning/datasource.yaml:/etc/grafana/provisioning/datasources/datasource.yaml:ro + - ./grafana/provisioning/dashboards.yaml:/etc/grafana/provisioning/dashboards/dashboards.yaml:ro + - ./grafana/dashboards:/var/lib/grafana/dashboards + networks: + - kagome-net + + +networks: + kagome-net: + name: kagome-net + external: true diff --git a/housekeeping/docker-compose/kagome/grafana/dashboards/Kagome _ Node-1733089320320.json b/housekeeping/docker-compose/kagome/grafana/dashboards/Kagome _ Node-1733089320320.json new file mode 100644 index 0000000000..332e9ed699 --- /dev/null +++ b/housekeeping/docker-compose/kagome/grafana/dashboards/Kagome _ Node-1733089320320.json @@ -0,0 +1,3966 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 39, + "panels": [], + "title": "Info", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "left", + "cellOptions": { + "type": "color-text" + }, + "filterable": false, + "inspect": true + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 11, + "x": 0, + "y": 1 + }, + "id": 37, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "enablePagination": false, + "fields": [], + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(kagome_build_info{}) by (name, chain, version)", + "format": "table", + "instant": true, + "legendFormat": "__auto", + "range": false, + "refId": "A" + } + ], + "title": "Build Info", + "transformations": [ + { + "id": "organize", + "options": { + "excludeByName": { + "Time": true, + "Value": true + }, + "indexByName": {}, + "renameByName": { + "chain": "Chain", + "version": "Version" + } + } + } + ], + "type": "table" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 11, + "y": 1 + }, + "id": 11, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "kagome_node_roles{}", + "format": "table", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "The roles the node is running as", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 15, + "y": 1 + }, + "id": 42, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "kagome_node_is_parachain_validator{}", + "format": "table", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Is Parachain Validator", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 18, + "y": 1 + }, + "id": 44, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "kagome_node_is_active_validator{}", + "format": "table", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Is Active Validator", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 21, + "y": 1 + }, + "id": 35, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "kagome_sub_libp2p_is_major_syncing{}", + "format": "table", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Major Syncing", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 5, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byRegexp", + "options": "/build.*/" + }, + "properties": [ + { + "id": "custom.lineWidth", + "value": 6 + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 54, + "options": { + "legend": { + "calcs": [], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "kagome_build_info{}", + "instant": false, + "legendFormat": "{{version}} ({{instance}})", + "range": true, + "refId": "A" + } + ], + "title": "Version and restarts", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 9 + }, + "id": 15, + "panels": [], + "title": "Node metrics", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlPu" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 10 + }, + "id": 28, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "time() - kagome_process_start_time_seconds{}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Process Uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-YlBl" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 21, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 10 + }, + "id": 4, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "time() - kagome_process_start_time_seconds{}", + "instant": false, + "legendFormat": "uptime", + "range": true, + "refId": "A" + } + ], + "title": "Process Uptime", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 14 + }, + "id": 16, + "panels": [], + "title": "Blocks", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-purples" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 0, + "y": 15 + }, + "id": 20, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(kagome_block_height{status=\"best\"})", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Block height info of the chain best", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-greens" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 8, + "y": 15 + }, + "id": 21, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "max(kagome_block_height{status=\"finalized\"})", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Block height info of the chain finalized", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlYlRd" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 16, + "y": 15 + }, + "id": 22, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kagome_block_height{status=\"best\"} - scalar(kagome_block_height{status=\"finalized\"}))", + "instant": false, + "legendFormat": "difference", + "range": true, + "refId": "A" + } + ], + "title": "Difference between best and finalized blocks", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 4, + "x": 20, + "y": 15 + }, + "id": 34, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "kagome_sub_libp2p_is_major_syncing{}", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Major Syncing", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-purples" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 0, + "y": 18 + }, + "id": 2, + "options": { + "legend": { + "calcs": [ + "last", + "min", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "kagome_block_height{status=\"best\"}", + "instant": false, + "legendFormat": "chain best", + "range": true, + "refId": "A" + } + ], + "title": "Block height info of the chain best", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 8, + "y": 18 + }, + "id": 8, + "options": { + "legend": { + "calcs": [ + "last", + "min", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "kagome_block_height{status=\"finalized\"}", + "instant": false, + "legendFormat": "chain finalized", + "range": true, + "refId": "A" + } + ], + "title": "Block height info of the chain finalized", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlPu" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 18 + }, + "id": 23, + "options": { + "legend": { + "calcs": [ + "last", + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.1.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "kagome_block_height{status=\"best\"} - scalar(kagome_block_height{status=\"finalized\"})", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "difference", + "range": true, + "refId": "A" + } + ], + "title": "Difference between best and finalized blocks", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-YlBl" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "sec/block", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 24 + }, + "id": 38, + "options": { + "legend": { + "calcs": [ + "max", + "min", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.1.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "000000001" + }, + "editorMode": "code", + "exemplar": true, + "expr": "1/rate(kagome_block_height{status=\"finalized\"}[5m])", + "hide": false, + "interval": "", + "legendFormat": "rate_of_new_blocks", + "range": true, + "refId": "A" + } + ], + "title": "Rate of new blocks", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 24, + "panels": [], + "title": "Status", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 6, + "x": 0, + "y": 31 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "kagome_finality_grandpa_round{}", + "instant": false, + "legendFormat": "kagome_finality_grandpa_round", + "range": true, + "refId": "A" + } + ], + "title": "Highest GRANDPA round", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlPu" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 6, + "y": 31 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "kagome_sync_peers{}", + "instant": false, + "legendFormat": "kagome_sync_peers", + "range": true, + "refId": "A" + } + ], + "title": "Number of peers we sync with", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "scaleDistribution": { + "type": "linear" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 9, + "x": 15, + "y": 31 + }, + "id": 7, + "options": { + "barRadius": 0, + "barWidth": 0.97, + "fullHighlight": false, + "groupWidth": 0.7, + "legend": { + "calcs": [ + "last", + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "orientation": "auto", + "showValue": "auto", + "stacking": "none", + "tooltip": { + "mode": "single", + "sort": "none" + }, + "xTickLabelRotation": 0, + "xTickLabelSpacing": 100 + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kagome_number_leaves{})", + "instant": false, + "legendFormat": "forks", + "range": true, + "refId": "A" + } + ], + "title": "Number of known chain leaves (aka forks)", + "type": "barchart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 39 + }, + "id": 19, + "panels": [], + "title": "RPC", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "purple", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 14, + "x": 0, + "y": 40 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kagome_rpc_sessions_opened{} - kagome_rpc_sessions_closed{})", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Number of actual persistent RPC sessions ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "green", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 14, + "y": 40 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "last", + "min", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kagome_rpc_sessions_opened{})", + "instant": false, + "legendFormat": "opened", + "range": true, + "refId": "A" + } + ], + "title": "Number of persistent RPC sessions opened", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "orange", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 19, + "y": 40 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "last", + "min", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kagome_rpc_sessions_closed{})", + "instant": false, + "legendFormat": "closed", + "range": true, + "refId": "A" + } + ], + "title": "Number of persistent RPC sessions closed", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 46 + }, + "id": 25, + "panels": [], + "title": "Parachain", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "valid" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "invalid" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 3, + "w": 8, + "x": 0, + "y": 47 + }, + "id": 27, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kagome_parachain_candidate_dispute_votes{validity=\"valid\"})", + "instant": false, + "legendFormat": "valid", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kagome_parachain_candidate_dispute_votes{validity=\"invalid\"})", + "hide": false, + "instant": false, + "legendFormat": "invalid", + "range": true, + "refId": "B" + } + ], + "title": "Parachain candidate disputes votes", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-BlPu", + "seriesBy": "max" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "blue", + "value": null + } + ] + }, + "unit": "bool" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 47 + }, + "id": 43, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "kagome_node_is_parachain_validator{}", + "instant": false, + "legendFormat": "kagome_node_is_parachain_validator", + "range": true, + "refId": "A" + } + ], + "title": "Is parachain validator", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 47 + }, + "id": 47, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kagome_parachain_availability_recovery_recoveries_started{})", + "instant": false, + "legendFormat": "recoveries started", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kagome_parachain_availability_recovery_recoveries_finished{})", + "hide": false, + "instant": false, + "legendFormat": "recoveries finished", + "range": true, + "refId": "B" + } + ], + "title": "Parachain availability recovery", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "valid" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "invalid" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 8, + "x": 0, + "y": 50 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kagome_parachain_candidate_dispute_votes{validity=\"valid\"})", + "instant": false, + "legendFormat": "valid", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kagome_parachain_candidate_dispute_votes{validity=\"invalid\"})", + "hide": false, + "instant": false, + "legendFormat": "invalid", + "range": true, + "refId": "B" + } + ], + "title": "Parachain candidate disputes votes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-YlBl", + "seriesBy": "max" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "area" + } + }, + "fieldMinMax": false, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bool" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 8, + "y": 52 + }, + "id": 45, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "kagome_node_is_active_validator{}", + "instant": false, + "legendFormat": "kagome_node_is_active_validator", + "range": true, + "refId": "A" + } + ], + "title": "Is active validator", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 57 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kagome_parachain_candidate_disputes_total{})", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Parachain candidate disputes total", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "yellow", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "valid" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "yellow", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "invalid" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "red", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 57 + }, + "id": 46, + "options": { + "legend": { + "calcs": [ + "lastNotNull", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kagome_parachain_approvals_no_shows_total{})", + "instant": false, + "legendFormat": "kagome_parachain_approvals_no_shows_total", + "range": true, + "refId": "A" + } + ], + "title": "Parachain approvals no shows", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "red", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 57 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "last", + "min", + "max", + "mean" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum (kagome_parachain_disputes_finality_lag{})", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Parachain finality disputes lag", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 65 + }, + "id": 49, + "panels": [], + "title": "Buckets", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 66 + }, + "id": 48, + "maxDataPoints": 60, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(kagome_pvf_execution_time_bucket{}[$__interval])) by (le)", + "format": "heatmap", + "hide": false, + "instant": false, + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "PVF execution time", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 66 + }, + "id": 50, + "maxDataPoints": 60, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(kagome_block_verification_and_import_time_bucket{}[$__interval])) by (le)", + "format": "heatmap", + "hide": false, + "instant": false, + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Block verification and import", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 66 + }, + "id": 52, + "maxDataPoints": 60, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(kagome_proposer_block_constructed_bucket{}[$__interval])) by (le)", + "format": "heatmap", + "hide": false, + "instant": false, + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Proposer block constructed", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 74 + }, + "id": 51, + "maxDataPoints": 60, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "s" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(kagome_pvf_preparation_time_bucket{}[$__interval])) by (le)", + "format": "heatmap", + "hide": false, + "instant": false, + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "PVF preparation time", + "type": "heatmap" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "scaleDistribution": { + "type": "linear" + } + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 74 + }, + "id": 53, + "maxDataPoints": 60, + "options": { + "calculate": false, + "cellGap": 1, + "color": { + "exponent": 0.5, + "fill": "dark-orange", + "mode": "scheme", + "reverse": false, + "scale": "exponential", + "scheme": "RdYlBu", + "steps": 64 + }, + "exemplars": { + "color": "rgba(255,0,255,0.7)" + }, + "filterValues": { + "le": 1e-9 + }, + "legend": { + "show": true + }, + "rowsFrame": { + "layout": "auto" + }, + "tooltip": { + "mode": "single", + "showColorScale": false, + "yHistogram": false + }, + "yAxis": { + "axisPlacement": "left", + "reverse": false, + "unit": "bytes" + } + }, + "pluginVersion": "10.4.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum(increase(kagome_parachain_candidate_validation_code_size_bucket{}[$__interval])) by (le)", + "format": "heatmap", + "hide": false, + "instant": false, + "legendFormat": "{{le}}", + "range": true, + "refId": "A" + } + ], + "title": "Parachain candidate validation code", + "type": "heatmap" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 82 + }, + "id": 55, + "panels": [], + "title": "Backed", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 83 + }, + "id": 56, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "sum(kagome_node_backed_candidates_in_block{})", + "instant": false, + "legendFormat": "kagome_node_backed_candidates_in_block", + "range": true, + "refId": "A" + } + ], + "title": "Backed Candidates In Block ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 83 + }, + "id": 57, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "kagome_node_include_candidates_in_block{}", + "instant": false, + "legendFormat": "kagome_node_include_candidates_in_block", + "range": true, + "refId": "A" + } + ], + "title": "Include Candidates In Block", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 83 + }, + "id": 58, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "kagome_node_no_backed_candidates_in_block{}", + "instant": false, + "legendFormat": "kagome_node_no_backed_candidates_in_block", + "range": true, + "refId": "A" + } + ], + "title": "No Backed Candidates In Block", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 0, + "y": 91 + }, + "id": 60, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "kagome_node_no_include_candidates_in_block{}", + "instant": false, + "legendFormat": "kagome_node_no_include_candidates_in_block", + "range": true, + "refId": "A" + } + ], + "title": "No Include Candidates In Block", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 8, + "y": 91 + }, + "id": 61, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "kagome_node_no_backed_no_include_candidates_in_block{}", + "instant": false, + "legendFormat": "kagome_node_no_backed_no_include_candidates_in_block", + "range": true, + "refId": "A" + } + ], + "title": "No Backed No Include Candidates In Block", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 50, + "gradientMode": "opacity", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 8, + "x": 16, + "y": 91 + }, + "id": 59, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prometheus" + }, + "editorMode": "code", + "expr": "kagome_node_total_blocks{}", + "instant": false, + "legendFormat": "kagome_node_total_blocks", + "range": true, + "refId": "A" + } + ], + "title": "Total number of blocks produced", + "type": "timeseries" + } + ], + "refresh": "5s", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Kagome | Node", + "uid": "dabed79e-2c7c-4df3-8ddd-bdd3ee14b1d0", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/housekeeping/docker-compose/kagome/grafana/provisioning/dashboards.yaml b/housekeeping/docker-compose/kagome/grafana/provisioning/dashboards.yaml new file mode 100644 index 0000000000..42664c7347 --- /dev/null +++ b/housekeeping/docker-compose/kagome/grafana/provisioning/dashboards.yaml @@ -0,0 +1,6 @@ +- name: 'default' + orgId: 1 + folder: '' + type: file + options: + path: /var/lib/grafana/dashboards diff --git a/housekeeping/docker-compose/kagome/grafana/provisioning/datasource.yaml b/housekeeping/docker-compose/kagome/grafana/provisioning/datasource.yaml new file mode 100644 index 0000000000..d7b8286865 --- /dev/null +++ b/housekeeping/docker-compose/kagome/grafana/provisioning/datasource.yaml @@ -0,0 +1,9 @@ +apiVersion: 1 + +datasources: +- name: Prometheus + type: prometheus + url: http://prometheus:9090 + isDefault: true + access: proxy + editable: true diff --git a/housekeeping/docker-compose/kagome/logcfg.yaml b/housekeeping/docker-compose/kagome/logcfg.yaml new file mode 100644 index 0000000000..c6dfeeea87 --- /dev/null +++ b/housekeeping/docker-compose/kagome/logcfg.yaml @@ -0,0 +1,129 @@ +sinks: + - name: console + type: console + stream: stdout + thread: name + color: true + latency: 0 + level: info + - name: logfile + type: file + path: /tmp/kagome/kagome.log + thread: name + capacity: 2048 + buffer: 4194304 + latency: 100 + level: info + - name: multi + type: multisink + sinks: + - console + - logfile +groups: + - name: main + sink: multi + level: info + is_fallback: true + children: + - name: libp2p + level: 'off' + - name: kagome + children: + - name: profile + - name: injector + - name: application + - name: rpc + children: + - name: rpc_transport + - name: api + children: + - name: author_api + - name: authorship + - name: blockchain + children: + - name: block_tree + - name: block_storage + - name: digest_tracker + - name: offchain + - name: authority + - name: crypto + children: + - name: bip39 + - name: key_store + - name: ed25519 + - name: ecdsa + - name: consensus + children: + - name: timeline + - name: babe + children: + - name: babe_lottery + - name: block_appender + - name: block_executor + level: info + - name: block_validator + - name: babe_config_repo + - name: grandpa + children: + - name: voting_round + - name: parachain + level: info + children: + - name: pvf_executor + - name: dispute + - name: runtime + children: + - name: runtime_api + - name: host_api + children: + - name: elliptic_curves_extension + - name: memory_extension + - name: io_extension + - name: crypto_extension + - name: storage_extension + - name: child_storage_extension + - name: offchain_extension + - name: misc_extension + - name: runtime_cache + - name: binaryen + - name: wavm + - name: wasmedge + - name: metrics + - name: telemetry + - name: network + children: + - name: reputation + - name: synchronizer + - name: authority_discovery + - name: kagome_protocols + children: + - name: block_announce_protocol + - name: grandpa_protocol + - name: propagate_transactions_protocol + - name: sync_protocol + - name: state_protocol + - name: warp_sync_protocol + - name: parachain_protocols + level: trace + children: + - name: collation_protocol_vstaging + - name: validation_protocol_vstaging + - name: req_collation_protocol + - name: req_chunk_protocol + - name: req_available_data_protocol + - name: req_statement_protocol + - name: req_pov_protocol + - name: dispute_protocol + - name: req_attested_candidate_protocol + - name: changes_trie + - name: storage + children: + - name: trie + - name: trie_pruner + - name: transactions + - name: pubsub + - name: threads + - name: others + children: + - name: testing + - name: debug \ No newline at end of file diff --git a/housekeeping/docker-compose/kagome/prometheus/config/prometheus.yaml b/housekeeping/docker-compose/kagome/prometheus/config/prometheus.yaml new file mode 100644 index 0000000000..4873dd59ab --- /dev/null +++ b/housekeeping/docker-compose/kagome/prometheus/config/prometheus.yaml @@ -0,0 +1,26 @@ +global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 15s + +scrape_configs: + - job_name: prometheus + honor_timestamps: true + scrape_interval: 15s + scrape_timeout: 10s + metrics_path: /metrics + scheme: http + static_configs: + - targets: + - localhost:9090 + + - job_name: kagome + honor_timestamps: true + scrape_interval: 5s + scrape_timeout: 3s + metrics_path: /metrics + scheme: http + static_configs: + - targets: + - kagome-node:9615 + - host.docker.internal:9615 diff --git a/scripts/.env b/scripts/.env index dbc079519e..bf42d5dbd6 100644 --- a/scripts/.env +++ b/scripts/.env @@ -5,4 +5,7 @@ CARGO_HOME=~/.cargo CMAKE_VERSION="3.25" CMAKE_CORE_NUMBER=6 +CXX="g++" +CC="gcc" + PATH="${CARGO_HOME}/bin:${PATH}" diff --git a/scripts/build.sh b/scripts/build.sh old mode 100644 new mode 100755 index dd99787a95..da0268d56f --- a/scripts/build.sh +++ b/scripts/build.sh @@ -1,10 +1,18 @@ #!/usr/bin/env bash + set -ex + current_dir=$(dirname $(readlink -f "$0")) parent_dir=$(dirname "$current_dir") -cd $parent_dir -set -a; source $current_dir/.env; set +a #include .env vars +set -a; source $current_dir/.env; set +a #include .env vars + +# install rust if not installed +if ! command -v rustc &> /dev/null +then + curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain ${RUST_VERSION} && \ + rustup default ${RUST_VERSION} +fi source $parent_dir/venv/bin/activate $parent_dir/venv/bin/cmake -H. -Bbuild -DCMAKE_BUILD_TYPE=Release diff --git a/scripts/init.sh b/scripts/init.sh index 11e3236acd..53de619d09 100755 --- a/scripts/init.sh +++ b/scripts/init.sh @@ -16,6 +16,3 @@ python3 -m venv "$parent_dir/venv" echo "Python environment created successfully in $parent_dir/venv" $parent_dir/venv/bin/pip install --no-cache-dir cmake==${CMAKE_VERSION} gitpython requests - -curl https://sh.rustup.rs -sSf | sh -s -- -y --default-toolchain ${RUST_VERSION} && \ - rustup default ${RUST_VERSION} From 6acdc454c2241f62bea5bb3cb0beb09d330b316a Mon Sep 17 00:00:00 2001 From: Alexander Lednev <57529355+iceseer@users.noreply.github.com> Date: Thu, 12 Dec 2024 15:04:11 +0300 Subject: [PATCH 15/17] [fix] deadlock in cb call (#2312) Signed-off-by: iceseer --- .../beefy_justification_protocol.cpp | 29 ++++++++++--------- .../protocols/fetch_attested_candidate.hpp | 6 ++-- core/network/impl/protocols/light.cpp | 23 ++++++++------- core/network/impl/protocols/light.hpp | 3 +- .../protocol_fetch_available_data.hpp | 13 +++++---- .../impl/protocols/protocol_fetch_chunk.hpp | 6 ++-- .../protocol_fetch_chunk_obsolete.hpp | 6 ++-- .../impl/protocols/protocol_req_collation.cpp | 19 ++++++++---- .../impl/protocols/protocol_req_collation.hpp | 4 ++- .../impl/protocols/protocol_req_pov.cpp | 16 ++++++---- .../impl/protocols/protocol_req_pov.hpp | 4 ++- .../protocols/request_response_protocol.hpp | 13 +++++++-- .../impl/protocols/send_dispute_protocol.hpp | 6 ++-- core/network/warp/protocol.hpp | 12 ++++---- .../availability/recovery/recovery_impl.cpp | 1 - 15 files changed, 102 insertions(+), 59 deletions(-) diff --git a/core/network/impl/protocols/beefy_justification_protocol.cpp b/core/network/impl/protocols/beefy_justification_protocol.cpp index 3d25e1b11d..ddab3bf822 100644 --- a/core/network/impl/protocols/beefy_justification_protocol.cpp +++ b/core/network/impl/protocols/beefy_justification_protocol.cpp @@ -16,20 +16,21 @@ namespace kagome::network { - BeefyJustificationProtocol::BeefyJustificationProtocol(libp2p::Host &host, - const blockchain::GenesisBlockHash &genesis, - common::MainThreadPool &main_thread_pool, - std::shared_ptr peer_manager, - std::shared_ptr beefy) - : RequestResponseProtocolImpl{ - kName, - host, - make_protocols(kBeefyJustificationProtocol, genesis), - log::createLogger(kName), - }, - main_pool_handler_{main_thread_pool.handlerStarted()}, - peer_manager_{std::move(peer_manager)}, - beefy_{std::move(beefy)} {} + BeefyJustificationProtocol::BeefyJustificationProtocol( + libp2p::Host &host, + const blockchain::GenesisBlockHash &genesis, + common::MainThreadPool &main_thread_pool, + std::shared_ptr peer_manager, + std::shared_ptr beefy) + : RequestResponseProtocolImpl{kName, + host, + make_protocols(kBeefyJustificationProtocol, + genesis), + log::createLogger(kName), + main_thread_pool}, + main_pool_handler_{main_thread_pool.handlerStarted()}, + peer_manager_{std::move(peer_manager)}, + beefy_{std::move(beefy)} {} std::optional> BeefyJustificationProtocol::onRxRequest(RequestType block, diff --git a/core/network/impl/protocols/fetch_attested_candidate.hpp b/core/network/impl/protocols/fetch_attested_candidate.hpp index 2ec93b0525..2c23e3b537 100644 --- a/core/network/impl/protocols/fetch_attested_candidate.hpp +++ b/core/network/impl/protocols/fetch_attested_candidate.hpp @@ -36,7 +36,8 @@ namespace kagome::network { const blockchain::GenesisBlockHash &genesis_hash, std::shared_ptr< parachain::statement_distribution::StatementDistribution> - statement_distribution) + statement_distribution, + common::MainThreadPool &main_thread_pool) : RequestResponseProtocolImpl< vstaging::AttestedCandidateRequest, vstaging::AttestedCandidateResponse, @@ -48,7 +49,8 @@ namespace kagome::network { kProtocolPrefixPolkadot), log::createLogger( kFetchAttestedCandidateProtocolName, - "req_attested_candidate_protocol")}, + "req_attested_candidate_protocol"), + main_thread_pool}, statement_distribution_(std::move(statement_distribution)) { BOOST_ASSERT(statement_distribution_); } diff --git a/core/network/impl/protocols/light.cpp b/core/network/impl/protocols/light.cpp index edd6740029..ce55d355ad 100644 --- a/core/network/impl/protocols/light.cpp +++ b/core/network/impl/protocols/light.cpp @@ -21,17 +21,18 @@ namespace kagome::network { std::shared_ptr repository, std::shared_ptr storage, std::shared_ptr module_repo, - std::shared_ptr executor) - : RequestResponseProtocolImpl{ - kName, - host, - make_protocols(kLightProtocol, genesis, chain_spec), - log::createLogger(kName), - }, - repository_{std::move(repository)}, - storage_{std::move(storage)}, - module_repo_{std::move(module_repo)}, - executor_{std::move(executor)} {} + std::shared_ptr executor, + common::MainThreadPool &main_thread_pool) + : RequestResponseProtocolImpl{kName, + host, + make_protocols( + kLightProtocol, genesis, chain_spec), + log::createLogger(kName), + main_thread_pool}, + repository_{std::move(repository)}, + storage_{std::move(storage)}, + module_repo_{std::move(module_repo)}, + executor_{std::move(executor)} {} std::optional> LightProtocol::onRxRequest(RequestType req, std::shared_ptr) { diff --git a/core/network/impl/protocols/light.hpp b/core/network/impl/protocols/light.hpp index 3620c9ae37..956b641a03 100644 --- a/core/network/impl/protocols/light.hpp +++ b/core/network/impl/protocols/light.hpp @@ -49,7 +49,8 @@ namespace kagome::network { std::shared_ptr repository, std::shared_ptr storage, std::shared_ptr module_repo, - std::shared_ptr executor); + std::shared_ptr executor, + common::MainThreadPool &main_thread_pool); std::optional> onRxRequest( RequestType req, std::shared_ptr) override; diff --git a/core/network/impl/protocols/protocol_fetch_available_data.hpp b/core/network/impl/protocols/protocol_fetch_available_data.hpp index 429c62eeb0..7790a380ed 100644 --- a/core/network/impl/protocols/protocol_fetch_available_data.hpp +++ b/core/network/impl/protocols/protocol_fetch_available_data.hpp @@ -33,7 +33,8 @@ namespace kagome::network { libp2p::Host &host, const application::ChainSpec &chain_spec, const blockchain::GenesisBlockHash &genesis_hash, - std::shared_ptr av_store) + std::shared_ptr av_store, + common::MainThreadPool &main_thread_pool) : RequestResponseProtocolImpl< FetchAvailableDataRequest, FetchAvailableDataResponse, @@ -44,8 +45,8 @@ namespace kagome::network { genesis_hash, kProtocolPrefixPolkadot), log::createLogger( - kName, - "req_available_data_protocol")}, + kName, "req_available_data_protocol"), + main_thread_pool}, av_store_{std::move(av_store)} {} private: @@ -77,7 +78,8 @@ namespace kagome::network { libp2p::Host &host, const application::ChainSpec &chain_spec, const blockchain::GenesisBlockHash &genesis_hash, - std::shared_ptr backing_store) + std::shared_ptr backing_store, + common::MainThreadPool &main_thread_pool) : RequestResponseProtocolImpl< FetchStatementRequest, FetchStatementResponse, @@ -87,7 +89,8 @@ namespace kagome::network { genesis_hash, kProtocolPrefixPolkadot), log::createLogger( - kName, "req_statement_protocol")}, + kName, "req_statement_protocol"), + main_thread_pool}, backing_store_{std::move(backing_store)} {} private: diff --git a/core/network/impl/protocols/protocol_fetch_chunk.hpp b/core/network/impl/protocols/protocol_fetch_chunk.hpp index 61fbf4440e..c477a9a343 100644 --- a/core/network/impl/protocols/protocol_fetch_chunk.hpp +++ b/core/network/impl/protocols/protocol_fetch_chunk.hpp @@ -43,7 +43,8 @@ namespace kagome::network { const application::ChainSpec & /*chain_spec*/, const blockchain::GenesisBlockHash &genesis_hash, std::shared_ptr pp, - std::shared_ptr pm) + std::shared_ptr pm, + common::MainThreadPool &main_thread_pool) : RequestResponseProtocolImpl< FetchChunkRequest, FetchChunkResponse, @@ -53,7 +54,8 @@ namespace kagome::network { genesis_hash, kProtocolPrefixPolkadot), log::createLogger(kFetchChunkProtocolName, - "req_chunk_protocol")}, + "req_chunk_protocol"), + main_thread_pool}, pp_{std::move(pp)}, pm_{std::move(pm)} { BOOST_ASSERT(pp_); diff --git a/core/network/impl/protocols/protocol_fetch_chunk_obsolete.hpp b/core/network/impl/protocols/protocol_fetch_chunk_obsolete.hpp index 5ba8a6ec77..087e665d6e 100644 --- a/core/network/impl/protocols/protocol_fetch_chunk_obsolete.hpp +++ b/core/network/impl/protocols/protocol_fetch_chunk_obsolete.hpp @@ -43,7 +43,8 @@ namespace kagome::network { libp2p::Host &host, const application::ChainSpec & /*chain_spec*/, const blockchain::GenesisBlockHash &genesis_hash, - std::shared_ptr pp) + std::shared_ptr pp, + common::MainThreadPool &main_thread_pool) : RequestResponseProtocolImpl< FetchChunkRequest, FetchChunkResponseObsolete, @@ -54,7 +55,8 @@ namespace kagome::network { genesis_hash, kProtocolPrefixPolkadot), log::createLogger(kFetchChunkProtocolName, - "req_chunk_protocol")}, + "req_chunk_protocol"), + main_thread_pool}, pp_{std::move(pp)} { BOOST_ASSERT(pp_); } diff --git a/core/network/impl/protocols/protocol_req_collation.cpp b/core/network/impl/protocols/protocol_req_collation.cpp index a2663d3ca6..ea85bd1657 100644 --- a/core/network/impl/protocols/protocol_req_collation.cpp +++ b/core/network/impl/protocols/protocol_req_collation.cpp @@ -29,12 +29,14 @@ namespace kagome::network { const libp2p::peer::ProtocolName &protoname, const application::ChainSpec &chain_spec, const blockchain::GenesisBlockHash &genesis_hash, - std::shared_ptr observer) + std::shared_ptr observer, + common::MainThreadPool &main_thread_pool) : Base{kReqCollationProtocolName, host, make_protocols(protoname, genesis_hash, kProtocolPrefixPolkadot), log::createLogger(kReqCollationProtocolName, - "req_collation_protocol")}, + "req_collation_protocol"), + main_thread_pool}, observer_{std::move(observer)} {} protected: @@ -60,11 +62,17 @@ namespace kagome::network { libp2p::Host &host, const application::ChainSpec &chain_spec, const blockchain::GenesisBlockHash &genesis_hash, - std::shared_ptr observer) + std::shared_ptr observer, + common::MainThreadPool &main_thread_pool) : v1_impl_{std::make_shared< ReqCollationProtocolImpl>( - host, kReqCollationProtocol, chain_spec, genesis_hash, observer)}, + host, + kReqCollationProtocol, + chain_spec, + genesis_hash, + observer, + main_thread_pool)}, vstaging_impl_{std::make_shared< ReqCollationProtocolImpl>( @@ -72,7 +80,8 @@ namespace kagome::network { kReqCollationVStagingProtocol, chain_spec, genesis_hash, - observer)} { + observer, + main_thread_pool)} { BOOST_ASSERT(v1_impl_); BOOST_ASSERT(vstaging_impl_); } diff --git a/core/network/impl/protocols/protocol_req_collation.hpp b/core/network/impl/protocols/protocol_req_collation.hpp index 6e97c23d84..e84faf452f 100644 --- a/core/network/impl/protocols/protocol_req_collation.hpp +++ b/core/network/impl/protocols/protocol_req_collation.hpp @@ -15,6 +15,7 @@ #include "application/app_configuration.hpp" #include "application/chain_spec.hpp" +#include "common/main_thread_pool.hpp" #include "log/logger.hpp" #include "network/peer_manager.hpp" #include "network/protocols/req_collation_protocol.hpp" @@ -41,7 +42,8 @@ namespace kagome::network { ReqCollationProtocol(libp2p::Host &host, const application::ChainSpec &chain_spec, const blockchain::GenesisBlockHash &genesis_hash, - std::shared_ptr observer); + std::shared_ptr observer, + common::MainThreadPool &main_thread_pool); const Protocol &protocolName() const override; diff --git a/core/network/impl/protocols/protocol_req_pov.cpp b/core/network/impl/protocols/protocol_req_pov.cpp index 75a97d1b91..44b2b5550e 100644 --- a/core/network/impl/protocols/protocol_req_pov.cpp +++ b/core/network/impl/protocols/protocol_req_pov.cpp @@ -23,7 +23,8 @@ namespace kagome::network { ReqPovProtocolImpl(libp2p::Host &host, const application::ChainSpec &chain_spec, const blockchain::GenesisBlockHash &genesis_hash, - std::shared_ptr observer) + std::shared_ptr observer, + common::MainThreadPool &main_thread_pool) : RequestResponseProtocolImpl< RequestPov, ResponsePov, @@ -33,7 +34,8 @@ namespace kagome::network { genesis_hash, kProtocolPrefixPolkadot), log::createLogger(kReqPovProtocolName, - "req_pov_protocol")}, + "req_pov_protocol"), + main_thread_pool}, observer_{std::move(observer)} {} protected: @@ -71,9 +73,13 @@ namespace kagome::network { libp2p::Host &host, const application::ChainSpec &chain_spec, const blockchain::GenesisBlockHash &genesis_hash, - std::shared_ptr observer) - : impl_{std::make_shared( - host, chain_spec, genesis_hash, std::move(observer))} {} + std::shared_ptr observer, + common::MainThreadPool &main_thread_pool) + : impl_{std::make_shared(host, + chain_spec, + genesis_hash, + std::move(observer), + main_thread_pool)} {} const Protocol &ReqPovProtocol::protocolName() const { BOOST_ASSERT(impl_ && !!"ReqPovProtocolImpl must be initialized!"); diff --git a/core/network/impl/protocols/protocol_req_pov.hpp b/core/network/impl/protocols/protocol_req_pov.hpp index 8ee8f62337..636b0e003e 100644 --- a/core/network/impl/protocols/protocol_req_pov.hpp +++ b/core/network/impl/protocols/protocol_req_pov.hpp @@ -15,6 +15,7 @@ #include "application/app_configuration.hpp" #include "application/chain_spec.hpp" +#include "common/main_thread_pool.hpp" #include "log/logger.hpp" #include "network/peer_manager.hpp" #include "network/protocols/req_pov_protocol.hpp" @@ -36,7 +37,8 @@ namespace kagome::network { ReqPovProtocol(libp2p::Host &host, const application::ChainSpec &chain_spec, const blockchain::GenesisBlockHash &genesis_hash, - std::shared_ptr observer); + std::shared_ptr observer, + common::MainThreadPool &main_thread_pool); const Protocol &protocolName() const override; diff --git a/core/network/impl/protocols/request_response_protocol.hpp b/core/network/impl/protocols/request_response_protocol.hpp index 326b8e3b32..8f72dd8635 100644 --- a/core/network/impl/protocols/request_response_protocol.hpp +++ b/core/network/impl/protocols/request_response_protocol.hpp @@ -8,6 +8,7 @@ #include "network/impl/protocols/protocol_base_impl.hpp" +#include "common/main_thread_pool.hpp" #include "protocol_error.hpp" #include "utils/box.hpp" @@ -39,9 +40,11 @@ namespace kagome::network { libp2p::Host &host, Protocols protocols, log::Logger logger, + common::MainThreadPool &main_thread_pool, std::chrono::milliseconds timeout = std::chrono::seconds(1)) : base_(std::move(name), host, std::move(protocols), std::move(logger)), - timeout_(std::move(timeout)) {} + timeout_(std::move(timeout)), + main_pool_handler_{main_thread_pool.handlerStarted()} {} bool start() override { return base_.start(this->weak_from_this()); @@ -125,7 +128,10 @@ namespace kagome::network { base_.host().getPeerRepository().getAddressRepository().getAddresses( peer_id); if (not addresses_res.has_value()) { - cb(addresses_res.as_failure()); + main_pool_handler_->execute( + [cb(std::move(cb)), addresses_res(std::move(addresses_res))] { + cb(addresses_res.as_failure()); + }); return; } @@ -374,6 +380,9 @@ namespace kagome::network { ProtocolBaseImpl base_; std::chrono::milliseconds timeout_; + + private: + std::shared_ptr main_pool_handler_; }; } // namespace kagome::network diff --git a/core/network/impl/protocols/send_dispute_protocol.hpp b/core/network/impl/protocols/send_dispute_protocol.hpp index e09f3e7f64..e809f31f34 100644 --- a/core/network/impl/protocols/send_dispute_protocol.hpp +++ b/core/network/impl/protocols/send_dispute_protocol.hpp @@ -45,7 +45,8 @@ namespace kagome::network { SendDisputeProtocolImpl(libp2p::Host &host, const blockchain::GenesisBlockHash &genesis_hash, std::shared_ptr - dispute_request_observer) + dispute_request_observer, + common::MainThreadPool &main_thread_pool) : RequestResponseProtocolImpl< DisputeRequest, DisputeResponse, @@ -56,7 +57,8 @@ namespace kagome::network { kProtocolPrefixPolkadot), log::createLogger( kSendDisputeProtocolName, - "dispute_protocol")}, + "dispute_protocol"), + main_thread_pool}, dispute_request_observer_{std::move(dispute_request_observer)} { BOOST_ASSERT(dispute_request_observer_); } diff --git a/core/network/warp/protocol.hpp b/core/network/warp/protocol.hpp index b8fba8b027..7c196b1857 100644 --- a/core/network/warp/protocol.hpp +++ b/core/network/warp/protocol.hpp @@ -30,12 +30,14 @@ namespace kagome::network { WarpProtocolImpl(libp2p::Host &host, const application::ChainSpec &chain_spec, const blockchain::GenesisBlockHash &genesis, - std::shared_ptr cache) + std::shared_ptr cache, + common::MainThreadPool &main_thread_pool) : RequestResponseProtocolImpl( - kName, - host, - make_protocols(kWarpProtocol, genesis, chain_spec), - log::createLogger(kName, "warp_sync_protocol")), + kName, + host, + make_protocols(kWarpProtocol, genesis, chain_spec), + log::createLogger(kName, "warp_sync_protocol"), + main_thread_pool), cache_{std::move(cache)} {} std::optional> onRxRequest( diff --git a/core/parachain/availability/recovery/recovery_impl.cpp b/core/parachain/availability/recovery/recovery_impl.cpp index bf5158b198..3f8004a43b 100644 --- a/core/parachain/availability/recovery/recovery_impl.cpp +++ b/core/parachain/availability/recovery/recovery_impl.cpp @@ -260,7 +260,6 @@ namespace kagome::parachain { candidate_hash, validator_index, peer->id); - lock.unlock(); send_fetch_available_data_request( peer->id, candidate_hash, From 1d5d75599f527abaa96ca54f6ca36eb00ba32236 Mon Sep 17 00:00:00 2001 From: Alexander Lednev <57529355+iceseer@users.noreply.github.com> Date: Fri, 13 Dec 2024 12:15:16 +0300 Subject: [PATCH 16/17] feature/scale v2 (#2289) * scale v2 Signed-off-by: iceseer --------- Signed-off-by: iceseer --- .../query/authority_peer_info.hpp | 2 +- core/authority_discovery/timestamp.hpp | 4 +- core/crypto/type_hasher.hpp | 7 +- .../helpers/scale_message_read_writer.hpp | 2 +- .../impl/sync_protocol_observer_impl.cpp | 2 +- core/network/types/collator_messages.hpp | 3 +- core/parachain/approval/knowledge.hpp | 5 +- core/parachain/pvf/pvf_impl.cpp | 2 +- core/parachain/types.hpp | 14 +- core/parachain/validator/signer.hpp | 2 +- .../statement_distribution/types.hpp | 2 +- core/primitives/inherent_data.hpp | 4 +- core/runtime/module_instance.hpp | 7 +- core/scale/encoder/concepts.hpp | 21 ++ core/scale/encoder/primitives.hpp | 334 ++++++++++-------- core/scale/kagome_scale.hpp | 295 +++++++++++----- core/utils/struct_to_tuple.hpp | 2 +- test/core/runtime/wasm_test.cpp | 70 +++- test/testutil/scale_test_comparator.hpp | 2 +- 19 files changed, 512 insertions(+), 268 deletions(-) create mode 100644 core/scale/encoder/concepts.hpp diff --git a/core/authority_discovery/query/authority_peer_info.hpp b/core/authority_discovery/query/authority_peer_info.hpp index 056da258ad..5393aa6d11 100644 --- a/core/authority_discovery/query/authority_peer_info.hpp +++ b/core/authority_discovery/query/authority_peer_info.hpp @@ -18,7 +18,7 @@ namespace kagome::authority_discovery { common::Buffer raw{}; std::optional time{}; - scale::PeerInfoSerializable peer{}; + ::scale::PeerInfoSerializable peer{}; }; } // namespace kagome::authority_discovery diff --git a/core/authority_discovery/timestamp.hpp b/core/authority_discovery/timestamp.hpp index f9cae7e204..d62d9cb057 100644 --- a/core/authority_discovery/timestamp.hpp +++ b/core/authority_discovery/timestamp.hpp @@ -9,6 +9,6 @@ #include "scale/big_fixed_integers.hpp" namespace kagome::authority_discovery { - using Timestamp = scale::uint128_t; - using TimestampScale = scale::Fixed; + using Timestamp = ::scale::uint128_t; + using TimestampScale = ::scale::Fixed; } // namespace kagome::authority_discovery diff --git a/core/crypto/type_hasher.hpp b/core/crypto/type_hasher.hpp index a3430e6055..96fa776772 100644 --- a/core/crypto/type_hasher.hpp +++ b/core/crypto/type_hasher.hpp @@ -15,8 +15,11 @@ namespace kagome::crypto { template inline void hashTypes(H &hasher, common::Blob &out, T &&...t) { - auto val = ::scale::encode(std::forward(t)...).value(); - hasher.update(val); + scale::encode( + [&](const uint8_t *const ptr, size_t count) { + hasher.update(std::span(ptr, count)); + }, + std::forward(t)...); hasher.get_final(out); } diff --git a/core/network/helpers/scale_message_read_writer.hpp b/core/network/helpers/scale_message_read_writer.hpp index d605d81848..8432a580e8 100644 --- a/core/network/helpers/scale_message_read_writer.hpp +++ b/core/network/helpers/scale_message_read_writer.hpp @@ -66,7 +66,7 @@ namespace kagome::network { template void write(const MsgType &msg, libp2p::basic::Writer::WriteCallbackFunc cb) const { - auto encoded_msg_res = ::scale::encode(msg); + auto encoded_msg_res = scale::encode(msg); if (!encoded_msg_res) { return cb(encoded_msg_res.error()); } diff --git a/core/network/impl/sync_protocol_observer_impl.cpp b/core/network/impl/sync_protocol_observer_impl.cpp index 50b802bf97..1eddcd0b7f 100644 --- a/core/network/impl/sync_protocol_observer_impl.cpp +++ b/core/network/impl/sync_protocol_observer_impl.cpp @@ -196,7 +196,7 @@ namespace kagome::network { if (auto r = beefy_->getJustification(*number)) { if (auto &opt = r.value()) { new_block.beefy_justification = primitives::Justification{ - common::Buffer{::scale::encode(*opt).value()}, + common::Buffer{scale::encode(*opt).value()}, }; } } diff --git a/core/network/types/collator_messages.hpp b/core/network/types/collator_messages.hpp index 0687abd3cd..625a9cdf7e 100644 --- a/core/network/types/collator_messages.hpp +++ b/core/network/types/collator_messages.hpp @@ -458,8 +458,7 @@ namespace kagome::network { auto commitments_hash = hasher.blake2b_256(scale::encode(receipt.commitments).value()); return hasher.blake2b_256( - ::scale::encode(std::tie(receipt.descriptor, commitments_hash)) - .value()); + scale::encode(std::tie(receipt.descriptor, commitments_hash)).value()); } inline CandidateHash candidateHash(const crypto::Hasher &hasher, diff --git a/core/parachain/approval/knowledge.hpp b/core/parachain/approval/knowledge.hpp index 9d6ff08890..5cb51df094 100644 --- a/core/parachain/approval/knowledge.hpp +++ b/core/parachain/approval/knowledge.hpp @@ -11,13 +11,14 @@ #include "common/visitor.hpp" #include "consensus/timeline/types.hpp" #include "outcome/outcome.hpp" +#include "parachain/approval/approval.hpp" #include "parachain/approval/state.hpp" #include "parachain/types.hpp" template <> struct std::hash { auto operator()(const scale::BitVec &v) const { - auto s = ::scale::encode(v).value(); + auto s = scale::encode(v).value(); return boost::hash_range(s.begin(), s.end()); } }; @@ -86,7 +87,7 @@ namespace kagome::parachain::approval { // Generate the knowledge keys for querying if an approval is known by peer. static std::pair generate_approval_key( - const approval::IndirectSignedApprovalVoteV2 &approval) { + const IndirectSignedApprovalVoteV2 &approval) { return { std::make_tuple(approval.payload.payload.block_hash, approval.payload.payload.candidate_indices, diff --git a/core/parachain/pvf/pvf_impl.cpp b/core/parachain/pvf/pvf_impl.cpp index 327a8128c6..882d11aad7 100644 --- a/core/parachain/pvf/pvf_impl.cpp +++ b/core/parachain/pvf/pvf_impl.cpp @@ -306,7 +306,7 @@ namespace kagome::parachain { receipt.descriptor.relay_parent, receipt.descriptor.para_id); - auto data_hash = hasher_->blake2b_256(::scale::encode(pvd).value()); + auto data_hash = hasher_->blake2b_256(scale::encode(pvd).value()); if (receipt.descriptor.persisted_data_hash != data_hash) { return cb(PvfError::PERSISTED_DATA_HASH); } diff --git a/core/parachain/types.hpp b/core/parachain/types.hpp index b2ea0e21e0..5d2c26ee4d 100644 --- a/core/parachain/types.hpp +++ b/core/parachain/types.hpp @@ -93,7 +93,7 @@ namespace kagome::parachain { auto signable() { constexpr std::array kMagic{'V', 'C', 'P', 'C'}; - return ::scale::encode(std::make_tuple(kMagic, *this)).value(); + return scale::encode(std::make_tuple(kMagic, *this)).value(); } }; } // namespace kagome::parachain @@ -171,11 +171,11 @@ namespace kagome::network { common::Buffer signable() const { return common::Buffer{ - ::scale::encode(relay_parent, - para_id, - persisted_data_hash, - pov_hash, - validation_code_hash) + scale::encode(relay_parent, + para_id, + persisted_data_hash, + pov_hash, + validation_code_hash) .value(), }; } @@ -193,7 +193,7 @@ namespace kagome::network { const parachain::Hash &hash(const crypto::Hasher &hasher) const { if (not hash_.has_value()) { hash_.emplace(hasher.blake2b_256( - ::scale::encode(std::tie(descriptor, commitments_hash)).value())); + scale::encode(std::tie(descriptor, commitments_hash)).value())); } return hash_.value(); } diff --git a/core/parachain/validator/signer.hpp b/core/parachain/validator/signer.hpp index 2151f0dce1..d264173e90 100644 --- a/core/parachain/validator/signer.hpp +++ b/core/parachain/validator/signer.hpp @@ -48,7 +48,7 @@ namespace kagome::parachain { template auto signable(const crypto::Hasher &hasher, const T &payload) const { auto &&signable = toSignable(hasher, payload); - return ::scale::encode(std::tie(signable, *this)).value(); + return scale::encode(std::tie(signable, *this)).value(); } /// Current session index. diff --git a/core/parachain/validator/statement_distribution/types.hpp b/core/parachain/validator/statement_distribution/types.hpp index fb20cf0367..ef4c890365 100644 --- a/core/parachain/validator/statement_distribution/types.hpp +++ b/core/parachain/validator/statement_distribution/types.hpp @@ -33,7 +33,7 @@ namespace kagome::parachain { statement, [&](const StatementWithPVDSeconded &val) { return hasher->blake2b_256( - ::scale::encode(val.committed_receipt.to_plain(*hasher)).value()); + scale::encode(val.committed_receipt.to_plain(*hasher)).value()); }, [&](const StatementWithPVDValid &val) { return val.candidate_hash; }); } diff --git a/core/primitives/inherent_data.hpp b/core/primitives/inherent_data.hpp index 5431f19bf4..2483ec94e0 100644 --- a/core/primitives/inherent_data.hpp +++ b/core/primitives/inherent_data.hpp @@ -50,7 +50,7 @@ namespace kagome::primitives { const T &inherent) { auto [it, inserted] = data.try_emplace(identifier, common::Buffer()); if (inserted) { - it->second = common::Buffer(::scale::encode(inherent).value()); + it->second = common::Buffer(scale::encode(inherent).value()); return outcome::success(); } return InherentDataError::IDENTIFIER_ALREADY_EXISTS; @@ -62,7 +62,7 @@ namespace kagome::primitives { */ template void replaceData(InherentIdentifier identifier, const T &inherent) { - data[identifier] = common::Buffer(::scale::encode(inherent).value()); + data[identifier] = common::Buffer(scale::encode(inherent).value()); } /** diff --git a/core/runtime/module_instance.hpp b/core/runtime/module_instance.hpp index 49e5d244db..1c2c10a26d 100644 --- a/core/runtime/module_instance.hpp +++ b/core/runtime/module_instance.hpp @@ -18,6 +18,7 @@ #include "outcome/outcome.hpp" #include "runtime/instance_environment.hpp" #include "runtime/ptr_size.hpp" +#include "scale/kagome_scale.hpp" namespace kagome::runtime { class Module; @@ -43,9 +44,9 @@ namespace kagome::runtime { template static outcome::result encodeArgs(const Args &...args) { if constexpr (sizeof...(args) > 0) { - return common::map_result(::scale::encode(args...), [](auto &&vec) { - return common::Buffer{vec}; - }); + return common::map_result( + kagome::scale::encode(args...), + [](auto &&vec) { return common::Buffer{vec}; }); } return outcome::success(); } diff --git a/core/scale/encoder/concepts.hpp b/core/scale/encoder/concepts.hpp new file mode 100644 index 0000000000..10d5800299 --- /dev/null +++ b/core/scale/encoder/concepts.hpp @@ -0,0 +1,21 @@ +/** + * Copyright Quadrivium LLC + * All Rights Reserved + * SPDX-License-Identifier: Apache-2.0 + */ + +#pragma once + +#include +#include + +namespace kagome::scale { + template + concept Invocable = std::is_invocable_v; + + template + concept IsEnum = std::is_enum_v>; + + template + concept IsNotEnum = !std::is_enum_v>; +} // namespace kagome::scale diff --git a/core/scale/encoder/primitives.hpp b/core/scale/encoder/primitives.hpp index eb48622c88..8b09d7619f 100644 --- a/core/scale/encoder/primitives.hpp +++ b/core/scale/encoder/primitives.hpp @@ -7,7 +7,10 @@ #ifndef KAGOME_SCALE_ENCODER_PRIMITIVES_HPP #define KAGOME_SCALE_ENCODER_PRIMITIVES_HPP +#include #include +#include +#include #include #include #include @@ -16,82 +19,90 @@ #include #include #include - +#include "crypto/ecdsa_types.hpp" +#include "primitives/math.hpp" +#include "scale/encoder/concepts.hpp" #include "utils/struct_to_tuple.hpp" namespace kagome::scale { + constexpr void putByte(const Invocable auto &func, + const uint8_t *const val, + size_t count); + + template + constexpr void encode(const Invocable auto &func, const std::tuple &v); - template - constexpr void putByte(const F &func, const uint8_t *const val, size_t count); + template + constexpr void encode(const Invocable auto &func, + const T &t, + const Args &...args); - template - constexpr void encode(const F &func, const std::tuple &v); + template + constexpr void encode(const Invocable auto &func, const std::vector &c); - template - constexpr void encode(const F &func, const T &t, const Args &...args); + template + constexpr void encode(const Invocable auto &func, const std::pair &p); - template - constexpr void encode(const F &func, const std::vector &c); + template + constexpr void encode(const Invocable auto &func, const std::span &c); - template - constexpr void encode(const FN &func, const std::pair &p); + template + constexpr void encode(const Invocable auto &func, const std::span &c); - template - constexpr void encode(const F &func, const std::span &c); + template + constexpr void encode(const Invocable auto &func, + const std::array &c); - template - constexpr void encode(const F &func, const std::span &c); + // NOLINTBEGIN + template + constexpr void encode(const Invocable auto &func, const T (&c)[N]); + // NOLINTEND - template - constexpr void encode(const F &func, const std::array &c); + template + constexpr void encode(const Invocable auto &func, const std::map &c); - template - constexpr void encode(const F &func, const T (&c)[N]); + template + constexpr void encode(const Invocable auto &func, + const std::shared_ptr &v); - template - constexpr void encode(const F &func, const std::map &c); + constexpr void encode(const Invocable auto &func, const std::string_view &v); - template - constexpr void encode(const F &func, const std::shared_ptr &v); + constexpr void encode(const Invocable auto &func, const std::string &v); - template - constexpr void encode(const F &func, const std::string_view &v); + template + constexpr void encode(const Invocable auto &func, + const std::unique_ptr &v); - template - constexpr void encode(const F &func, const std::string &v); + template + constexpr void encode(const Invocable auto &func, const std::list &c); - template - constexpr void encode(const F &func, const std::unique_ptr &v); + template + constexpr void encode(const Invocable auto &func, const std::deque &c); - template - constexpr void encode(const F &func, const std::list &c); + template + void encode(const Invocable auto &func, const boost::variant &v); - template - constexpr void encode(const F &func, const std::deque &c); + template + void encode(const Invocable auto &func, const boost::variant &v); - template - void encode(const F &func, const boost::variant &v); + template + void encode(const Invocable auto &func, const std::variant &v); - template - void encode(const F &func, const boost::variant &v); + void encode(const Invocable auto &func, const ::scale::CompactInteger &value); - template - void encode(const F &func, const ::scale::CompactInteger &value); + void encode(const Invocable auto &func, const ::scale::BitVec &value); - template - void encode(const F &func, const ::scale::BitVec &value); + void encode(const Invocable auto &func, const std::optional &value); - template - void encode(const F &func, const std::optional &value); + template + void encode(const Invocable auto &func, const std::optional &value); - template - void encode(const F &func, const std::optional &value); + void encode(const Invocable auto &func, const crypto::EcdsaSignature &value); - template >, bool> = true> - constexpr void encode(const F &func, const T &v) { - using I = std::decay_t; + void encode(const Invocable auto &func, const crypto::EcdsaPublicKey &value); + + constexpr void encode(const Invocable auto &func, const IsNotEnum auto &v) { + using I = std::decay_t; if constexpr (std::is_integral_v) { if constexpr (std::is_same_v) { const uint8_t byte = (v ? 1u : 0u); @@ -108,26 +119,37 @@ namespace kagome::scale { const auto val = math::toLE(v); putByte(func, (uint8_t *)&val, size); } else { - encode(func, utils::to_tuple_refs(v)); + kagome::scale::encode(func, utils::to_tuple_refs(v)); } } - template >, bool> = true> - constexpr void encode(const F &func, const T &value) { - encode(func, static_cast>>(value)); + constexpr void encode(const Invocable auto &func, const IsEnum auto &value) { + kagome::scale::encode( + func, + static_cast>>( + value)); } - template - constexpr void encode(const F &func, const T &t, const Args &...args) { - encode(func, t); - encode(func, args...); + template + constexpr void encode(const Invocable auto &func, + const T &t, + const Args &...args) { + kagome::scale::encode(func, t); + kagome::scale::encode(func, args...); } + // NOLINTBEGIN(cppcoreguidelines-pro-bounds-pointer-arithmetic) template outcome::result> encode(const Args &...args) { - return ::scale::encode(args...); + std::vector res; + kagome::scale::encode( + [&](const uint8_t *const val, size_t count) { + if (count != 0ull) { + res.insert(res.end(), &val[0], &val[count]); + } + }, + args...); + return res; } inline size_t bitUpperBorder(const ::scale::CompactInteger &x) { @@ -168,71 +190,71 @@ namespace kagome::scale { } return counter; } + // NOLINTEND(cppcoreguidelines-pro-bounds-pointer-arithmetic) - template - constexpr void putByte(const F &func, + constexpr void putByte(const Invocable auto &func, const uint8_t *const val, size_t count) { func(val, count); } - template - void encode(const F &func, const boost::variant &v) { + template + void encode(const Invocable auto &func, const boost::variant &v) { using T = std::tuple_element_t>; if (v.which() == I) { - encode(func, I); - encode(func, boost::get(v)); + kagome::scale::encode(func, I); + kagome::scale::encode(func, boost::get(v)); return; } if constexpr (sizeof...(Ts) > I + 1) { - encode(func, v); + kagome::scale::encode(func, v); } } - template - constexpr void encode(const F &func, const T (&c)[N]) { + // NOLINTBEGIN + template + constexpr void encode(const Invocable auto &func, const T (&c)[N]) { using E = std::decay_t; if constexpr (std::is_integral_v && sizeof(E) == 1u) { putByte(func, c, N); } else { for (const auto &e : c) { - encode(func, e); + kagome::scale::encode(func, e); } } } + // NOLINTEND - template - void encode(const F &func, const boost::variant &v) { - encode(func, v); + template + void encode(const Invocable auto &func, const boost::variant &v) { + kagome::scale::encode<0>(func, v); } - template , std::enable_if_t, bool> = true> - constexpr void encodeCompactSmall(const F &func, T val) { + constexpr void encodeCompactSmall(const Invocable auto &func, T val) { BOOST_ASSERT_MSG((val >> (8 * sizeof(I) - 2)) == 0, "Unexpected compact value in encoder"); val <<= 2; val |= (sizeof(I) / 2ull); - encode(func, val); + kagome::scale::encode(func, val); } - template - void encodeCompact(const F &func, uint64_t val) { + void encodeCompact(const Invocable auto &func, uint64_t val) { if (val < ::scale::compact::EncodingCategoryLimits::kMinUint16) { - encodeCompactSmall(func, static_cast(val)); + kagome::scale::encodeCompactSmall(func, static_cast(val)); return; } if (val < ::scale::compact::EncodingCategoryLimits::kMinUint32) { - encodeCompactSmall(func, static_cast(val)); + kagome::scale::encodeCompactSmall(func, static_cast(val)); return; } if (val < ::scale::compact::EncodingCategoryLimits::kMinBigInteger) { - encodeCompactSmall(func, static_cast(val)); + kagome::scale::encodeCompactSmall(func, static_cast(val)); return; } @@ -245,24 +267,27 @@ namespace kagome::scale { putByte(func, result, bigIntLength + 1ull); } - template - constexpr void encode(const F &func, const std::string &v) { - encode(func, std::string_view{v}); + template + void encode(const Invocable auto &func, const std::variant &v) { + kagome::scale::encode(func, (uint8_t)v.index()); + std::visit([&](const auto &s) { kagome::scale::encode(func, s); }, v); + } + + constexpr void encode(const Invocable auto &func, const std::string &v) { + kagome::scale::encode(func, std::string_view{v}); } - template - constexpr void encode(const F &func, const std::string_view &v) { - encodeCompact(func, v.size()); + constexpr void encode(const Invocable auto &func, const std::string_view &v) { + kagome::scale::encodeCompact(func, v.size()); putByte(func, (const uint8_t *)v.data(), v.size()); } - template - void encode(const F &func, const ::scale::BitVec &v) { + void encode(const Invocable auto &func, const ::scale::BitVec &v) { const size_t bitsCount = v.bits.size(); const size_t bytesCount = ((bitsCount + 7ull) >> 3ull); const size_t blocksCount = ((bytesCount + 7ull) >> 3ull); - encodeCompact(func, bitsCount); + kagome::scale::encodeCompact(func, bitsCount); uint64_t result; size_t bitCounter = 0ull; for (size_t ix = 0ull; ix < blocksCount; ++ix) { @@ -282,25 +307,25 @@ namespace kagome::scale { } } - template - void encode(const F &func, const ::scale::CompactInteger &value) { + void encode(const Invocable auto &func, + const ::scale::CompactInteger &value) { if (value < 0) { raise(::scale::EncodeError::NEGATIVE_COMPACT_INTEGER); } const size_t bit_border = bitUpperBorder(value); if (bit_border <= 6) { // kMinUint16 - encodeCompactSmall(func, value.convert_to()); + kagome::scale::encodeCompactSmall(func, value.convert_to()); return; } if (bit_border <= 14) { // kMinUint32 - encodeCompactSmall(func, value.convert_to()); + kagome::scale::encodeCompactSmall(func, value.convert_to()); return; } if (bit_border <= 30) { // kMinBigInteger - encodeCompactSmall(func, value.convert_to()); + kagome::scale::encodeCompactSmall(func, value.convert_to()); return; } @@ -329,28 +354,27 @@ namespace kagome::scale { } template < - typename F, typename It, typename = std::enable_if_t< !std::is_same_v::value_type, void>>> - constexpr void encode(const F &func, It begin, It end) { + constexpr void encode(const Invocable auto &func, It begin, It end) { while (begin != end) { - encode(func, *begin); + kagome::scale::encode(func, *begin); ++begin; } } - template - constexpr void encode(const F &func, const std::span &c) { - encodeCompact(func, c.size()); - encode(func, c.begin(), c.end()); + template + constexpr void encode(const Invocable auto &func, const std::span &c) { + kagome::scale::encodeCompact(func, c.size()); + kagome::scale::encode(func, c.begin(), c.end()); } - template - constexpr void encode(const F &func, const std::span &c) { + template + constexpr void encode(const Invocable auto &func, const std::span &c) { if constexpr (S == -1) { - encodeCompact(func, c.size()); - encode(func, c.begin(), c.end()); + kagome::scale::encodeCompact(func, c.size()); + kagome::scale::encode(func, c.begin(), c.end()); } else { using E = std::decay_t; if constexpr (std::is_integral_v && sizeof(E) == 1u) { @@ -363,68 +387,72 @@ namespace kagome::scale { } } - template - constexpr void encode(const F &func, const std::array &c) { + template + constexpr void encode(const Invocable auto &func, + const std::array &c) { for (const auto &e : c) { - encode(func, e); + kagome::scale::encode(func, e); } } - template - constexpr void encode(const F &func, const std::map &c) { - encodeCompact(func, c.size()); - encode(func, c.begin(), c.end()); + template + constexpr void encode(const Invocable auto &func, const std::map &c) { + kagome::scale::encodeCompact(func, c.size()); + kagome::scale::encode(func, c.begin(), c.end()); } - template - constexpr void encode(const FN &func, const std::pair &p) { - encode(func, p.first); - encode(func, p.second); + template + constexpr void encode(const Invocable auto &func, const std::pair &p) { + kagome::scale::encode(func, p.first); + kagome::scale::encode(func, p.second); } - template - constexpr void encode(const F &func, const std::vector &c) { - encodeCompact(func, c.size()); - encode(func, c.begin(), c.end()); + template + constexpr void encode(const Invocable auto &func, const std::vector &c) { + kagome::scale::encodeCompact(func, c.size()); + kagome::scale::encode(func, c.begin(), c.end()); } - template - constexpr void encode(const F &func, const std::shared_ptr &v) { + template + constexpr void encode(const Invocable auto &func, + const std::shared_ptr &v) { if (v == nullptr) { raise(::scale::EncodeError::DEREF_NULLPOINTER); } - encode(func, *v); + kagome::scale::encode(func, *v); } - template - constexpr void encode(const F &func, const std::unique_ptr &v) { + template + constexpr void encode(const Invocable auto &func, + const std::unique_ptr &v) { if (v == nullptr) { raise(::scale::EncodeError::DEREF_NULLPOINTER); } - encode(func, *v); + kagome::scale::encode(func, *v); } - template - constexpr void encode(const F &func, const std::list &c) { - encodeCompact(func, c.size()); - encode(func, c.begin(), c.end()); + template + constexpr void encode(const Invocable auto &func, const std::list &c) { + kagome::scale::encodeCompact(func, c.size()); + kagome::scale::encode(func, c.begin(), c.end()); } - template - constexpr void encode(const F &func, const std::deque &c) { - encodeCompact(func, c.size()); - encode(func, c.begin(), c.end()); + template + constexpr void encode(const Invocable auto &func, const std::deque &c) { + kagome::scale::encodeCompact(func, c.size()); + kagome::scale::encode(func, c.begin(), c.end()); } - template - constexpr void encode(const F &func, const std::tuple &v) { + template + constexpr void encode(const Invocable auto &func, + const std::tuple &v) { if constexpr (sizeof...(Ts) > 0) { - std::apply([&](const auto &...s) { (..., encode(func, s)); }, v); + std::apply( + [&](const auto &...s) { (..., kagome::scale::encode(func, s)); }, v); } } - template - void encode(const F &func, const std::optional &v) { + void encode(const Invocable auto &func, const std::optional &v) { enum class OptionalBool : uint8_t { NONE = 0u, OPT_TRUE = 1u, @@ -437,19 +465,29 @@ namespace kagome::scale { } else if (!*v) { result = OptionalBool::OPT_FALSE; } - encode(func, result); + kagome::scale::encode(func, result); } - template - void encode(const F &func, const std::optional &v) { + template + void encode(const Invocable auto &func, const std::optional &v) { if (!v.has_value()) { - encode(func, uint8_t(0u)); + kagome::scale::encode(func, uint8_t(0u)); } else { - encode(func, uint8_t(1u)); - encode(func, *v); + kagome::scale::encode(func, uint8_t(1u)); + kagome::scale::encode(func, *v); } } + void encode(const Invocable auto &func, const crypto::EcdsaSignature &data) { + kagome::scale::encode( + func, static_cast(data)); + } + + void encode(const Invocable auto &func, const crypto::EcdsaPublicKey &data) { + kagome::scale::encode( + func, static_cast(data)); + } + } // namespace kagome::scale #endif // KAGOME_SCALE_ENCODER_PRIMITIVES_HPP diff --git a/core/scale/kagome_scale.hpp b/core/scale/kagome_scale.hpp index fde89b7b95..aa1c90d5a7 100644 --- a/core/scale/kagome_scale.hpp +++ b/core/scale/kagome_scale.hpp @@ -12,7 +12,11 @@ #include "common/blob.hpp" #include "consensus/babe/types/babe_block_header.hpp" #include "consensus/babe/types/seal.hpp" +#include "consensus/beefy/types.hpp" +#include "consensus/grandpa/types/equivocation_proof.hpp" #include "network/types/blocks_response.hpp" +#include "network/types/collator_messages_vstaging.hpp" +#include "network/types/dispute_messages.hpp" #include "network/types/roles.hpp" #include "primitives/block_header.hpp" #include "primitives/block_id.hpp" @@ -20,6 +24,7 @@ #include "runtime/runtime_api/parachain_host_types.hpp" #include "scale/big_fixed_integers.hpp" #include "scale/encode_append.hpp" +#include "scale/encoder/concepts.hpp" #include "scale/libp2p_types.hpp" #include @@ -39,166 +44,255 @@ namespace kagome::scale { using ::scale::decode; - template - constexpr void encode(const F &func, const primitives::BlockHeader &bh); + constexpr void encode(const Invocable auto &func, + const primitives::BlockHeader &bh); - template - constexpr void encode(const F &func, const network::BlocksResponse &b); + constexpr void encode(const Invocable auto &func, + const consensus::grandpa::Equivocation &bh); + + constexpr void encode(const Invocable auto &func, + const primitives::BlockHeaderReflection &bh); + + constexpr void encode(const Invocable auto &func, + const primitives::BlockReflection &bh); + + constexpr void encode(const Invocable auto &func, + const network::BlocksResponse &b); - template + constexpr void encode(const Invocable auto &func, + const kagome::network::vstaging::CompactStatement &c); + + template constexpr void encode( - const F &func, + const Invocable auto &func, const common::SLVector &c); - template - constexpr void encode(const F &func, const Tagged &c); + template + constexpr void encode(const Invocable auto &func, + const Tagged &c); - template - constexpr void encode(const F &func, const common::SLBuffer &c); + template + constexpr void encode(const Invocable auto &func, + const common::SLBuffer &c); - template - constexpr void encode(const F &func, const network::Roles &c); + constexpr void encode(const Invocable auto &func, const network::Roles &c); - template - constexpr void encode(const F &func, const primitives::Other &c); + constexpr void encode(const Invocable auto &func, const primitives::Other &c); - template - constexpr void encode(const F &func, const primitives::Consensus &c); + constexpr void encode(const Invocable auto &func, + const primitives::Consensus &c); - template - constexpr void encode(const F &func, + constexpr void encode(const Invocable auto &func, const runtime::PersistedValidationData &c); - template - constexpr void encode(const F &func, const primitives::Seal &c); + constexpr void encode(const Invocable auto &func, const primitives::Seal &c); - template - constexpr void encode(const F &func, const primitives::PreRuntime &c); + constexpr void encode(const Invocable auto &func, + const primitives::PreRuntime &c); - template - constexpr void encode(const F &func, const primitives::BlockInfo &c); + constexpr void encode(const Invocable auto &func, + const primitives::BlockInfo &c); - template - constexpr void encode(const F &func, + constexpr void encode(const Invocable auto &func, const primitives::RuntimeEnvironmentUpdated &c); - template - constexpr void encode(const F &func, const ::scale::EncodeOpaqueValue &c); + constexpr void encode(const Invocable auto &func, + const ::scale::EncodeOpaqueValue &c); - template - constexpr void encode(const F &func, + constexpr void encode(const Invocable auto &func, const consensus::babe::BabeBlockHeader &bh); + constexpr void encode(const Invocable auto &func, + const kagome::network::CandidateCommitments &c); + + constexpr void encode(const Invocable auto &func, + const kagome::network::CandidateReceipt &c); + + constexpr void encode(const Invocable auto &func, + const kagome::network::InvalidDisputeVote &c); + + constexpr void encode(const Invocable auto &func, + const kagome::network::ValidDisputeVote &c); + + constexpr void encode(const Invocable auto &func, + const consensus::grandpa::SignedPrecommit &c); template constexpr void encode(const F &func, const authority_discovery::AuthorityPeerInfo &c); + constexpr void encode(const Invocable auto &func, + const scale::PeerInfoSerializable &c); + + template + constexpr void encode(const Invocable auto &func, const Fixed &c); } // namespace kagome::scale #include "scale/encoder/primitives.hpp" namespace kagome::scale { - template - constexpr void encode(const F &func, const primitives::BlockHeader &bh) { - encode(func, bh.parent_hash); - encodeCompact(func, bh.number); - encode(func, bh.state_root); - encode(func, bh.extrinsics_root); - encode(func, bh.digest); + constexpr void encode(const Invocable auto &func, + const primitives::BlockHeader &bh) { + kagome::scale::encode(func, bh.parent_hash); + kagome::scale::encodeCompact(func, bh.number); + kagome::scale::encode(func, bh.state_root); + kagome::scale::encode(func, bh.extrinsics_root); + kagome::scale::encode(func, bh.digest); } - template - constexpr void encode(const F &func, const network::BlocksResponse &b) { - encode(func, b.blocks); + constexpr void encode(const Invocable auto &func, + const primitives::BlockReflection &bh) { + kagome::scale::encode(func, bh.header); + kagome::scale::encode(func, bh.body); } - template - constexpr void encode(const F &func, + constexpr void encode(const Invocable auto &func, + const primitives::BlockHeaderReflection &bh) { + kagome::scale::encode(func, bh.parent_hash); + kagome::scale::encodeCompact(func, bh.number); + kagome::scale::encode(func, bh.state_root); + kagome::scale::encode(func, bh.extrinsics_root); + kagome::scale::encode(func, bh.digest); + } + + constexpr void encode(const Invocable auto &func, + const network::BlocksResponse &b) { + kagome::scale::encode(func, b.blocks); + } + + constexpr void encode(const Invocable auto &func, const consensus::babe::BabeBlockHeader &bh) { - encode(func, bh.slot_assignment_type); - encode(func, bh.authority_index); - encode(func, bh.slot_number); + kagome::scale::encode(func, bh.slot_assignment_type); + kagome::scale::encode(func, bh.authority_index); + kagome::scale::encode(func, bh.slot_number); if (bh.needVRFCheck()) { - encode(func, bh.vrf_output); + kagome::scale::encode(func, bh.vrf_output); } } - template + template constexpr void encode( - const F &func, + const Invocable auto &func, const common::SLVector &c) { - encode(func, static_cast &>(c)); + kagome::scale::encode( + func, static_cast &>(c)); } - template - constexpr void encode(const F &func, const Tagged &c) { + template + constexpr void encode(const Invocable auto &func, + const Tagged &c) { if constexpr (std::is_scalar_v) { - encode(func, c.Wrapper::value); + kagome::scale::encode(func, c.Wrapper::value); } else { - encode(func, static_cast(c)); + kagome::scale::encode(func, static_cast(c)); } } - template - constexpr void encode(const F &func, const common::SLBuffer &c) { - encode(func, static_cast &>(c)); + template + constexpr void encode(const Invocable auto &func, + const common::SLBuffer &c) { + kagome::scale::encode( + func, static_cast &>(c)); } - template - constexpr void encode(const F &func, const primitives::Other &c) { - encode(func, static_cast(c)); + constexpr void encode(const Invocable auto &func, + const primitives::Other &c) { + kagome::scale::encode(func, static_cast(c)); } - template - constexpr void encode(const F &func, const primitives::Consensus &c) { - encode(func, static_cast(c)); + constexpr void encode(const Invocable auto &func, + const primitives::Consensus &c) { + kagome::scale::encode( + func, static_cast(c)); } - template - constexpr void encode(const F &func, + constexpr void encode(const Invocable auto &func, const kagome::runtime::PersistedValidationData &c) { - encode(func, c.parent_head); - encode(func, c.relay_parent_number); - encode(func, c.relay_parent_storage_root); - encode(func, c.max_pov_size); + kagome::scale::encode(func, c.parent_head); + kagome::scale::encode(func, c.relay_parent_number); + kagome::scale::encode(func, c.relay_parent_storage_root); + kagome::scale::encode(func, c.max_pov_size); } - template - constexpr void encode(const F &func, const primitives::Seal &c) { - encode(func, static_cast(c)); + constexpr void encode(const Invocable auto &func, const primitives::Seal &c) { + kagome::scale::encode( + func, static_cast(c)); } - template - constexpr void encode(const F &func, const primitives::PreRuntime &c) { - encode(func, static_cast(c)); + constexpr void encode(const Invocable auto &func, + const primitives::PreRuntime &c) { + kagome::scale::encode( + func, static_cast(c)); } - template - constexpr void encode(const F &func, const primitives::BlockInfo &c) { - encode(func, c.number); - encode(func, c.hash); + constexpr void encode(const Invocable auto &func, + const primitives::BlockInfo &c) { + kagome::scale::encode(func, c.number); + kagome::scale::encode(func, c.hash); } - template - constexpr void encode(const F &func, + constexpr void encode(const Invocable auto &func, const primitives::RuntimeEnvironmentUpdated &c) {} - template - constexpr void encode(const F &func, const ::scale::EncodeOpaqueValue &c) { + constexpr void encode(const Invocable auto &func, + const ::scale::EncodeOpaqueValue &c) { putByte(func, c.v.data(), c.v.size()); } - template - constexpr void encode(const F &func, const network::Roles &c) { - encode(func, c.value); + constexpr void encode(const Invocable auto &func, const network::Roles &c) { + kagome::scale::encode( + func, c.value); // NOLINT(cppcoreguidelines-pro-type-union-access) + } + + constexpr void encode(const Invocable auto &func, + const consensus::grandpa::Equivocation &bh) { + kagome::scale::encode(func, bh.stage); + kagome::scale::encode(func, bh.round_number); + kagome::scale::encode(func, bh.first); + kagome::scale::encode(func, bh.second); + } + + constexpr void encode(const Invocable auto &func, + const kagome::network::CandidateCommitments &c) { + kagome::scale::encode(func, c.upward_msgs); + kagome::scale::encode(func, c.outbound_hor_msgs); + kagome::scale::encode(func, c.opt_para_runtime); + kagome::scale::encode(func, c.para_head); + kagome::scale::encode(func, c.downward_msgs_count); + kagome::scale::encode(func, c.watermark); + } + + constexpr void encode(const Invocable auto &func, + const kagome::network::CandidateReceipt &c) { + kagome::scale::encode(func, c.descriptor); + kagome::scale::encode(func, c.commitments_hash); + } + + constexpr void encode(const Invocable auto &func, + const kagome::network::vstaging::CompactStatement &c) { + kagome::scale::encode(func, c.header); + kagome::scale::encode(func, c.inner_value); + } + + constexpr void encode(const Invocable auto &func, + const kagome::network::InvalidDisputeVote &c) { + kagome::scale::encode(func, c.index); + kagome::scale::encode(func, c.signature); + kagome::scale::encode(func, c.kind); + } + + constexpr void encode(const Invocable auto &func, + const kagome::network::ValidDisputeVote &c) { + kagome::scale::encode(func, c.index); + kagome::scale::encode(func, c.signature); + kagome::scale::encode(func, c.kind); + } + + constexpr void encode(const Invocable auto &func, + const consensus::grandpa::SignedPrecommit &c) { + kagome::scale::encode( + func, static_cast(c)); } template @@ -209,6 +303,25 @@ namespace kagome::scale { encode(func, c.peer); } + constexpr void encode(const Invocable auto &func, + const scale::PeerInfoSerializable &c) { + std::vector addresses; + addresses.reserve(c.addresses.size()); + for (const auto &addr : c.addresses) { + addresses.emplace_back(addr.getStringAddress()); + } + encode(func, c.id.toBase58()); + encode(func, addresses); + } + + template + constexpr void encode(const Invocable auto &func, const Fixed &c) { + constexpr size_t bits = Fixed::kByteSize * 8; + for (size_t i = 0; i < bits; i += 8) { + encode(func, ::scale::convert_to((*c >> i) & 0xFFu)); + } + } + } // namespace kagome::scale #endif // KAGOME_KAGOME_SCALE_HPP diff --git a/core/utils/struct_to_tuple.hpp b/core/utils/struct_to_tuple.hpp index 0f43ba7929..03bf5189b2 100644 --- a/core/utils/struct_to_tuple.hpp +++ b/core/utils/struct_to_tuple.hpp @@ -75,7 +75,7 @@ namespace kagome::utils { template - decltype(void(T{{std::declval()}...}), std::true_type{}) + decltype(void(T{std::declval()...}), std::true_type()) test_is_braces_constructible(int); template diff --git a/test/core/runtime/wasm_test.cpp b/test/core/runtime/wasm_test.cpp index 1bc5bbc524..6bbffefdf4 100644 --- a/test/core/runtime/wasm_test.cpp +++ b/test/core/runtime/wasm_test.cpp @@ -1,4 +1,72 @@ -#include +#include +#include + +/// #include Conflicting with fmt::ranges::formatter +namespace qtils { + struct Hex { + qtils::BytesIn v; + }; +} // namespace qtils + +template <> +struct fmt::formatter { + bool prefix = true; + bool full = false; + bool lower = true; + + constexpr auto parse(format_parse_context &ctx) { + auto it = ctx.begin(); + auto end = [&] { return it == ctx.end() or *it == '}'; }; + if (end()) { + return it; + } + prefix = *it == '0'; + if (prefix) { + ++it; + } + if (not end()) { + lower = *it == 'x'; + if (lower or *it == 'X') { + ++it; + full = true; + if (end()) { + return it; + } + } + } + fmt::throw_format_error("\"x\"/\"X\" or \"0x\"/\"0X\" expected"); + } + + auto format(const qtils::BytesIn &bytes, format_context &ctx) const { + auto out = ctx.out(); + if (prefix) { + out = fmt::detail::write(out, "0x"); + } + constexpr size_t kHead = 2, kTail = 2, kSmall = 1; + if (full or bytes.size() <= kHead + kTail + kSmall) { + return lower ? fmt::format_to(out, "{:02x}", fmt::join(bytes, "")) + : fmt::format_to(out, "{:02X}", fmt::join(bytes, "")); + } + return fmt::format_to(out, + "{:02x}…{:02x}", + fmt::join(bytes.first(kHead), ""), + fmt::join(bytes.last(kTail), "")); + } +}; + +template <> +struct fmt::formatter : fmt::formatter {}; + +template <> +struct fmt::formatter : fmt::formatter {}; + +template <> +struct fmt::formatter : fmt::formatter { + auto format(const qtils::Hex &v, format_context &ctx) const { + return formatter::format(v.v, ctx); + } +}; + #include #include "common/bytestr.hpp" #include "crypto/hasher/hasher_impl.hpp" diff --git a/test/testutil/scale_test_comparator.hpp b/test/testutil/scale_test_comparator.hpp index ad2f4cf578..fede908b0b 100644 --- a/test/testutil/scale_test_comparator.hpp +++ b/test/testutil/scale_test_comparator.hpp @@ -14,7 +14,7 @@ namespace testutil { template inline outcome::result> scaleEncodeAndCompareWithRef( T &&...t) { - return ::scale::encode(std::forward(t)...); + return scale::encode(std::forward(t)...); } } // namespace testutil From d056f23bd79bf099c12c916b50e21b3ae296c27a Mon Sep 17 00:00:00 2001 From: Ruslan Tushov Date: Mon, 16 Dec 2024 07:58:12 +0500 Subject: [PATCH 17/17] pvf priority (#2305) Signed-off-by: turuslan Co-authored-by: kamilsa --- core/parachain/pvf/pvf_impl.cpp | 1 + core/parachain/pvf/workers.cpp | 41 +++++++++++++++++++++++--------- core/parachain/pvf/workers.hpp | 12 ++++++++-- core/storage/rocksdb/rocksdb.cpp | 2 +- 4 files changed, 42 insertions(+), 14 deletions(-) diff --git a/core/parachain/pvf/pvf_impl.cpp b/core/parachain/pvf/pvf_impl.cpp index 882d11aad7..cdec26202a 100644 --- a/core/parachain/pvf/pvf_impl.cpp +++ b/core/parachain/pvf/pvf_impl.cpp @@ -391,6 +391,7 @@ namespace kagome::parachain { } cb(scale::decode(r.value())); }, + .kind = timeout_kind, .timeout = std::chrono::milliseconds{ timeout_kind == runtime::PvfExecTimeoutKind::Backing diff --git a/core/parachain/pvf/workers.cpp b/core/parachain/pvf/workers.cpp index 3842cc0759..5551676a6d 100644 --- a/core/parachain/pvf/workers.cpp +++ b/core/parachain/pvf/workers.cpp @@ -20,6 +20,8 @@ #include "utils/weak_macro.hpp" namespace kagome::parachain { + constexpr auto kMetricQueueSize = "kagome_pvf_queue_size"; + struct AsyncPipe : boost::process::async_pipe { using async_pipe::async_pipe; using lowest_layer_type = AsyncPipe; @@ -133,13 +135,26 @@ namespace kagome::parachain { .cache_dir = app_config.runtimeCacheDirPath(), .log_params = app_config.log(), .force_disable_secure_mode = app_config.disableSecureMode(), - } {} + } { + metrics_registry_->registerGaugeFamily(kMetricQueueSize, "pvf queue size"); + std::unordered_map kind_name{ + {PvfExecTimeoutKind::Approval, "Approval"}, + {PvfExecTimeoutKind::Backing, "Backing"}, + }; + for (auto &[kind, name] : kind_name) { + metric_queue_size_.emplace(kind, + metrics_registry_->registerGaugeMetric( + kMetricQueueSize, {{"kind", name}})); + } + } void PvfWorkers::execute(Job &&job) { REINVOKE(*main_pool_handler_, execute, std::move(job)); if (free_.empty()) { if (used_ >= max_) { - queue_.emplace(std::move(job)); + auto &queue = queues_[job.kind]; + queue.emplace_back(std::move(job)); + metric_queue_size_.at(job.kind)->set(queue.size()); return; } auto used = std::make_shared(*this); @@ -157,10 +172,9 @@ namespace kagome::parachain { if (not r) { return job.cb(r.error()); } - self->writeCode( - std::move(job), - {.process = std::move(process)}, - std::move(used)); + self->writeCode(std::move(job), + {.process = std::move(process)}, + std::move(used)); }); return; } @@ -244,11 +258,16 @@ namespace kagome::parachain { } void PvfWorkers::dequeue() { - if (queue_.empty()) { - return; + for (auto &kind : + {PvfExecTimeoutKind::Approval, PvfExecTimeoutKind::Backing}) { + auto &queue = queues_[kind]; + if (queue.empty()) { + continue; + } + auto job = std::move(queue.front()); + queue.pop_front(); + metric_queue_size_.at(kind)->set(queue.size()); + findFree(std::move(job)); } - auto job = std::move(queue_.front()); - queue_.pop(); - findFree(std::move(job)); } } // namespace kagome::parachain diff --git a/core/parachain/pvf/workers.hpp b/core/parachain/pvf/workers.hpp index b267e73cd9..b9dcbde5ad 100644 --- a/core/parachain/pvf/workers.hpp +++ b/core/parachain/pvf/workers.hpp @@ -6,11 +6,13 @@ #pragma once +#include #include #include -#include +#include "metrics/metrics.hpp" #include "parachain/pvf/pvf_worker_types.hpp" +#include "runtime/runtime_api/parachain_host_types.hpp" namespace boost::asio { class io_context; @@ -33,6 +35,8 @@ namespace kagome::common { } // namespace kagome::common namespace kagome::parachain { + using runtime::PvfExecTimeoutKind; + struct ProcessAndPipes; class PvfWorkers : public std::enable_shared_from_this { @@ -46,6 +50,7 @@ namespace kagome::parachain { PvfWorkerInputCodeParams code_params; Buffer args; Cb cb; + PvfExecTimeoutKind kind; std::chrono::milliseconds timeout{0}; }; void execute(Job &&job); @@ -77,6 +82,9 @@ namespace kagome::parachain { PvfWorkerInputConfig worker_config_; std::list free_; size_t used_ = 0; - std::queue queue_; + std::unordered_map> queues_; + + metrics::RegistryPtr metrics_registry_ = metrics::createRegistry(); + std::unordered_map metric_queue_size_; }; } // namespace kagome::parachain diff --git a/core/storage/rocksdb/rocksdb.cpp b/core/storage/rocksdb/rocksdb.cpp index 8e1623026c..104050e18f 100644 --- a/core/storage/rocksdb/rocksdb.cpp +++ b/core/storage/rocksdb/rocksdb.cpp @@ -93,7 +93,7 @@ namespace kagome::storage { if (not enable_migration) { SL_ERROR(log, "Database migration is disabled, use older kagome version or " - "run with migration enabling flag"); + "run with --enable-db-migration flag"); return DatabaseError::IO_ERROR; }