Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/backing tests #2315

Open
wants to merge 43 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
43 commits
Select commit Hold shift + click to select a range
5125a47
[feature] interface for StatementDistribution
iceseer Dec 1, 2024
9eb135c
[feature] IPeerView
iceseer Dec 1, 2024
42b1040
[feature] IBitfieldSigner
iceseer Dec 1, 2024
d4b4cd4
[feature] IValidatorSignerFactory
iceseer Dec 1, 2024
faf43fa
[feature] IPvfPrecheck
iceseer Dec 1, 2024
3892286
[feature] IProspectiveParachains
iceseer Dec 1, 2024
71c1e80
formatting
iceseer Dec 1, 2024
15c4449
backing tests
iceseer Dec 2, 2024
362c5e7
[feature] peerviewmock
iceseer Dec 3, 2024
962d959
[feature] BitfieldSignerMock
iceseer Dec 3, 2024
c401eca
[feature] PvfPrecheckMock
iceseer Dec 3, 2024
91ddee8
[feature] pvf_mock
iceseer Dec 3, 2024
89e97c4
[feature] signer_factory_mock
iceseer Dec 3, 2024
3eab5d0
[feature] building
iceseer Dec 3, 2024
6aed23f
bind
iceseer Dec 3, 2024
6a5a476
init
iceseer Dec 3, 2024
7699262
tests
iceseer Dec 4, 2024
2bbd6e4
tests
iceseer Dec 4, 2024
f78611a
formatting
iceseer Dec 4, 2024
87db410
tests
iceseer Dec 4, 2024
8a86690
tests
iceseer Dec 5, 2024
db6d8c3
validator signer
iceseer Dec 5, 2024
1c36a70
signing context
iceseer Dec 5, 2024
8d0dbf8
formatting
iceseer Dec 5, 2024
ac40ec1
builded
iceseer Dec 5, 2024
e6c78b6
fixup
iceseer Dec 5, 2024
74e5285
trace 2
iceseer Dec 6, 2024
a40381c
[feature] threaded `parachain processor`
iceseer Dec 17, 2024
e6c3b10
formatting
iceseer Dec 17, 2024
32c12d3
[refactor] `ParachainProcessor` interface
iceseer Dec 17, 2024
d018efa
interface
iceseer Dec 18, 2024
4f259e0
[TEST] ParachainProcessorEmpty
iceseer Dec 18, 2024
75d1964
Merge commit 'd056f23bd79bf099c12c916b50e21b3ae296c27a' into feature/…
iceseer Dec 18, 2024
b96de09
fix
iceseer Dec 18, 2024
00cc4a5
[fix] test output fixup
iceseer Jan 9, 2025
000cf19
[fixup] relay parent correct
iceseer Jan 9, 2025
17530f2
Merge remote-tracking branch 'origin/master' into feature/backing_tests
iceseer Jan 9, 2025
7332abd
[tests] `backing_test` first test complete
iceseer Jan 13, 2025
f2cee61
[kagome] multithreaded parachain processor
iceseer Jan 13, 2025
3919cff
clear logs
iceseer Jan 14, 2025
a7b37d9
formatting
iceseer Jan 14, 2025
c3e10a8
Merge branch 'master' into feature/backing_tests
iceseer Jan 14, 2025
35b5e33
remove comments
iceseer Jan 14, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion core/consensus/babe/impl/babe.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ namespace kagome::offchain {

namespace kagome::parachain {
class BitfieldStore;
class ParachainProcessorImpl;
class ParachainProcessor;
struct BackedCandidatesSource;
} // namespace kagome::parachain

Expand Down
34 changes: 30 additions & 4 deletions core/injector/application_injector.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -114,6 +114,7 @@
#include "metrics/impl/metrics_watcher.hpp"
#include "metrics/impl/prometheus/handler_impl.hpp"
#include "metrics/metrics.hpp"
#include "network/i_peer_view.hpp"
#include "network/impl/block_announce_transmitter_impl.hpp"
#include "network/impl/extrinsic_observer_impl.hpp"
#include "network/impl/grandpa_transmitter_impl.hpp"
Expand Down Expand Up @@ -152,6 +153,7 @@
#include "outcome/outcome.hpp"
#include "parachain/approval/approval_distribution.hpp"
#include "parachain/approval/approval_thread_pool.hpp"
#include "parachain/availability/bitfield/signer.hpp"
#include "parachain/availability/bitfield/store_impl.hpp"
#include "parachain/availability/fetch/fetch_impl.hpp"
#include "parachain/availability/recovery/recovery_impl.hpp"
Expand All @@ -164,6 +166,7 @@
#include "parachain/pvf/workers.hpp"
#include "parachain/validator/impl/parachain_observer_impl.hpp"
#include "parachain/validator/parachain_processor.hpp"
#include "parachain/validator/statement_distribution/i_statement_distribution.hpp"
#include "parachain/validator/statement_distribution/statement_distribution.hpp"
#include "runtime/binaryen/binaryen_memory_provider.hpp"
#include "runtime/binaryen/instance_environment_factory.hpp"
Expand Down Expand Up @@ -784,12 +787,35 @@ namespace {
di::bind<network::SyncProtocolObserver>.template to<network::SyncProtocolObserverImpl>(),
di::bind<network::DisputeRequestObserver>.template to<dispute::DisputeCoordinatorImpl>(),
di::bind<parachain::AvailabilityStore>.template to<parachain::AvailabilityStoreImpl>(),
di::bind<network::IPeerView>.template to<network::PeerView>(),
di::bind<parachain::IBitfieldSigner>.template to<parachain::BitfieldSigner>(),
di::bind<parachain::Fetch>.template to<parachain::FetchImpl>(),
di::bind<parachain::Recovery>.template to<parachain::RecoveryImpl>(),
di::bind<parachain::IValidatorSignerFactory>.template to<parachain::ValidatorSignerFactory>(),
di::bind<parachain::BitfieldStore>.template to<parachain::BitfieldStoreImpl>(),
di::bind<parachain::BackingStore>.template to<parachain::BackingStoreImpl>(),
di::bind<parachain::BackedCandidatesSource>.template to<parachain::ParachainProcessorImpl>(),
di::bind<network::CanDisconnect>.template to<parachain::statement_distribution::StatementDistribution>(),
di::bind<parachain::IProspectiveParachains>.template to<parachain::ProspectiveParachains>(),
bind_by_lambda<parachain::BackedCandidatesSource>(
[](const auto &injector) {
return injector.template create<sptr<parachain::ThreadedParachainProcessorImpl>>();
}),
bind_by_lambda<parachain::ParachainStorage>(
[](const auto &injector) {
return injector.template create<sptr<parachain::ThreadedParachainProcessorImpl>>();
}),
bind_by_lambda<parachain::ParachainProcessor>(
[](const auto &injector) {
return injector.template create<sptr<parachain::ThreadedParachainProcessorImpl>>();
}),
di::bind<parachain::IPvfPrecheck>.template to<parachain::PvfPrecheck>(),
bind_by_lambda<network::CanDisconnect>(
[](const auto &injector) {
return injector.template create<sptr<parachain::statement_distribution::StatementDistribution>>();
}),
bind_by_lambda<parachain::statement_distribution::IStatementDistribution>(
[](const auto &injector) {
return injector.template create<sptr<parachain::statement_distribution::StatementDistribution>>();
}),
di::bind<parachain::Pvf>.template to<parachain::PvfImpl>(),
bind_by_lambda<parachain::SecureModeSupport>([config](const auto &) {
auto support = parachain::SecureModeSupport::none();
Expand Down Expand Up @@ -1034,10 +1060,10 @@ namespace kagome::injector {
.template create<sptr<parachain::ParachainObserver>>();
}

std::shared_ptr<parachain::ParachainProcessorImpl>
std::shared_ptr<parachain::ParachainProcessor>
KagomeNodeInjector::injectParachainProcessor() {
return pimpl_->injector_
.template create<sptr<parachain::ParachainProcessorImpl>>();
.template create<sptr<parachain::ThreadedParachainProcessorImpl>>();
}

std::shared_ptr<parachain::statement_distribution::StatementDistribution>
Expand Down
8 changes: 4 additions & 4 deletions core/injector/application_injector.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -59,12 +59,13 @@ namespace kagome {

namespace parachain {
class ParachainObserver;
class ParachainProcessorImpl;
class ParachainProcessor;
class ApprovalDistribution;

namespace statement_distribution {
class StatementDistribution;
}
class IStatementDistribution;
} // namespace statement_distribution
} // namespace parachain

namespace runtime {
Expand Down Expand Up @@ -138,8 +139,7 @@ namespace kagome::injector {
std::shared_ptr<network::SyncProtocolObserver> injectSyncObserver();
std::shared_ptr<network::StateProtocolObserver> injectStateObserver();
std::shared_ptr<parachain::ParachainObserver> injectParachainObserver();
std::shared_ptr<parachain::ParachainProcessorImpl>
injectParachainProcessor();
std::shared_ptr<parachain::ParachainProcessor> injectParachainProcessor();
std::shared_ptr<parachain::statement_distribution::StatementDistribution>
injectStatementDistribution();
std::shared_ptr<parachain::ApprovalDistribution>
Expand Down
69 changes: 69 additions & 0 deletions core/network/i_peer_view.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
/**
* Copyright Quadrivium LLC
* All Rights Reserved
* SPDX-License-Identifier: Apache-2.0
*/
#pragma once

#include <memory>
#include <unordered_map>

#include <libp2p/peer/peer_id.hpp>

#include "crypto/type_hasher.hpp"
#include "network/types/collator_messages.hpp"
#include "outcome/outcome.hpp"
#include "primitives/event_types.hpp"
#include "subscription/subscriber.hpp"
#include "subscription/subscription_engine.hpp"

namespace kagome::network {

using HashedBlockHeader = primitives::BlockHeader;
struct ExView {
View view;
HashedBlockHeader new_head;
std::vector<primitives::BlockHash> lost;
};

struct ExViewRef {
std::optional<std::reference_wrapper<const HashedBlockHeader>> new_head;
const std::vector<primitives::BlockHash> &lost;
};

/**
* Observable class for current heads and finalized block number tracking.
*/
class IPeerView {
public:
enum struct EventType : uint32_t { kViewUpdated, kPeerRemoved };

using PeerId = libp2p::peer::PeerId;

using MyViewSubscriptionEngine = subscription::
SubscriptionEngine<EventType, std::monostate, network::ExView>;
using MyViewSubscriptionEnginePtr =
std::shared_ptr<MyViewSubscriptionEngine>;
using MyViewSubscriber = MyViewSubscriptionEngine::SubscriberType;
using MyViewSubscriberPtr = std::shared_ptr<MyViewSubscriber>;

using PeerViewSubscriptionEngine = subscription::
SubscriptionEngine<EventType, std::monostate, PeerId, network::View>;
using PeerViewSubscriptionEnginePtr =
std::shared_ptr<PeerViewSubscriptionEngine>;
using PeerViewSubscriber = PeerViewSubscriptionEngine::SubscriberType;
using PeerViewSubscriberPtr = std::shared_ptr<PeerViewSubscriber>;

virtual ~IPeerView() = default;

virtual size_t peersCount() const = 0;
virtual MyViewSubscriptionEnginePtr getMyViewObservable() = 0;
virtual PeerViewSubscriptionEnginePtr getRemoteViewObservable() = 0;

virtual void removePeer(const PeerId &peer_id) = 0;
virtual void updateRemoteView(const PeerId &peer_id,
network::View &&view) = 0;
virtual const View &getMyView() const = 0;
};

} // namespace kagome::network
15 changes: 7 additions & 8 deletions core/network/impl/protocols/protocol_fetch_chunk.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -38,13 +38,12 @@ namespace kagome::network {
NonCopyable,
NonMovable {
public:
FetchChunkProtocolImpl(
libp2p::Host &host,
const application::ChainSpec & /*chain_spec*/,
const blockchain::GenesisBlockHash &genesis_hash,
std::shared_ptr<parachain::ParachainProcessorImpl> pp,
std::shared_ptr<PeerManager> pm,
common::MainThreadPool &main_thread_pool)
FetchChunkProtocolImpl(libp2p::Host &host,
const application::ChainSpec & /*chain_spec*/,
const blockchain::GenesisBlockHash &genesis_hash,
std::shared_ptr<parachain::ParachainStorage> pp,
std::shared_ptr<PeerManager> pm,
common::MainThreadPool &main_thread_pool)
: RequestResponseProtocolImpl<
FetchChunkRequest,
FetchChunkResponse,
Expand Down Expand Up @@ -132,7 +131,7 @@ namespace kagome::network {
}

inline static const auto kFetchChunkProtocolName = "FetchChunkProtocol_v2"s;
std::shared_ptr<parachain::ParachainProcessorImpl> pp_;
std::shared_ptr<parachain::ParachainStorage> pp_;
std::shared_ptr<PeerManager> pm_;
};

Expand Down
4 changes: 2 additions & 2 deletions core/network/impl/protocols/protocol_fetch_chunk_obsolete.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ namespace kagome::network {
libp2p::Host &host,
const application::ChainSpec & /*chain_spec*/,
const blockchain::GenesisBlockHash &genesis_hash,
std::shared_ptr<parachain::ParachainProcessorImpl> pp,
std::shared_ptr<parachain::ParachainStorage> pp,
common::MainThreadPool &main_thread_pool)
: RequestResponseProtocolImpl<
FetchChunkRequest,
Expand Down Expand Up @@ -117,7 +117,7 @@ namespace kagome::network {
}

inline static const auto kFetchChunkProtocolName = "FetchChunkProtocol_v1"s;
std::shared_ptr<parachain::ParachainProcessorImpl> pp_;
std::shared_ptr<parachain::ParachainStorage> pp_;
};

} // namespace kagome::network
56 changes: 8 additions & 48 deletions core/network/peer_view.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -5,20 +5,10 @@
*/
#pragma once

#include <memory>
#include <unordered_map>

#include <libp2p/peer/peer_id.hpp>

#include "application/app_state_manager.hpp"
#include "blockchain/block_tree.hpp"
#include "crypto/type_hasher.hpp"
#include "injector/lazy.hpp"
#include "network/types/collator_messages.hpp"
#include "outcome/outcome.hpp"
#include "primitives/event_types.hpp"
#include "subscription/subscriber.hpp"
#include "subscription/subscription_engine.hpp"
#include "network/i_peer_view.hpp"
#include "utils/non_copyable.hpp"
#include "utils/safe_object.hpp"

Expand All @@ -28,43 +18,14 @@ namespace kagome::blockchain {

namespace kagome::network {

using HashedBlockHeader = primitives::BlockHeader;
struct ExView {
View view;
HashedBlockHeader new_head;
std::vector<primitives::BlockHash> lost;
};

struct ExViewRef {
std::optional<std::reference_wrapper<const HashedBlockHeader>> new_head;
const std::vector<primitives::BlockHash> &lost;
};

/**
* Observable class for current heads and finalized block number tracking.
*/
class PeerView final : public NonCopyable,
public NonMovable,
public IPeerView,
public std::enable_shared_from_this<PeerView> {
public:
enum struct EventType : uint32_t { kViewUpdated, kPeerRemoved };

using PeerId = libp2p::peer::PeerId;

using MyViewSubscriptionEngine = subscription::
SubscriptionEngine<EventType, std::monostate, network::ExView>;
using MyViewSubscriptionEnginePtr =
std::shared_ptr<MyViewSubscriptionEngine>;
using MyViewSubscriber = MyViewSubscriptionEngine::SubscriberType;
using MyViewSubscriberPtr = std::shared_ptr<MyViewSubscriber>;

using PeerViewSubscriptionEngine = subscription::
SubscriptionEngine<EventType, std::monostate, PeerId, network::View>;
using PeerViewSubscriptionEnginePtr =
std::shared_ptr<PeerViewSubscriptionEngine>;
using PeerViewSubscriber = PeerViewSubscriptionEngine::SubscriberType;
using PeerViewSubscriberPtr = std::shared_ptr<PeerViewSubscriber>;

PeerView(primitives::events::ChainSubscriptionEnginePtr chain_events_engine,
std::shared_ptr<application::AppStateManager> app_state_manager,
LazySPtr<blockchain::BlockTree> block_tree);
Expand All @@ -76,14 +37,13 @@ namespace kagome::network {
bool prepare();
void stop();

size_t peersCount() const;

MyViewSubscriptionEnginePtr getMyViewObservable();
PeerViewSubscriptionEnginePtr getRemoteViewObservable();
size_t peersCount() const override;
MyViewSubscriptionEnginePtr getMyViewObservable() override;
PeerViewSubscriptionEnginePtr getRemoteViewObservable() override;

void removePeer(const PeerId &peer_id);
void updateRemoteView(const PeerId &peer_id, network::View &&view);
auto &getMyView() const {
void removePeer(const PeerId &peer_id) override;
void updateRemoteView(const PeerId &peer_id, network::View &&view) override;
const View &getMyView() const override {
return my_view_;
}

Expand Down
14 changes: 14 additions & 0 deletions core/parachain/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,18 @@ target_link_libraries(backing_implicit_view
outcome
)

add_library(parachain_errors
validator/i_parachain_processor.cpp
)

target_link_libraries(parachain_errors
fmt::fmt
soralog::soralog
logger
Boost::boost
outcome
)

add_library(validator_parachain
availability/bitfield/signer.cpp
availability/bitfield/store_impl.cpp
Expand All @@ -54,6 +66,7 @@ add_library(validator_parachain
backing/store_impl.cpp
backing/cluster.cpp
validator/statement_distribution/statement_distribution.cpp
validator/parachain_storage.cpp
)

target_link_libraries(validator_parachain
Expand All @@ -68,6 +81,7 @@ target_link_libraries(validator_parachain
runtime_common
prospective_parachains
backing_implicit_view
parachain_errors
)

add_library(kagome_pvf_worker
Expand Down
2 changes: 1 addition & 1 deletion core/parachain/approval/approval_distribution.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -577,7 +577,7 @@ namespace kagome::parachain {
std::shared_ptr<crypto::KeyStore> keystore,
std::shared_ptr<crypto::Hasher> hasher,
std::shared_ptr<network::PeerView> peer_view,
std::shared_ptr<ParachainProcessorImpl> parachain_processor,
std::shared_ptr<ParachainProcessor> parachain_processor,
std::shared_ptr<crypto::Sr25519Provider> crypto_provider,
std::shared_ptr<network::PeerManager> pm,
std::shared_ptr<network::Router> router,
Expand Down
6 changes: 3 additions & 3 deletions core/parachain/approval/approval_distribution.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,7 @@ namespace kagome::consensus::babe {

namespace kagome::parachain {
class ApprovalThreadPool;
class ParachainProcessorImpl;
class ParachainProcessor;
class Pvf;
} // namespace kagome::parachain

Expand Down Expand Up @@ -300,7 +300,7 @@ namespace kagome::parachain {
std::shared_ptr<crypto::KeyStore> keystore,
std::shared_ptr<crypto::Hasher> hasher,
std::shared_ptr<network::PeerView> peer_view,
std::shared_ptr<ParachainProcessorImpl> parachain_processor,
std::shared_ptr<ParachainProcessor> parachain_processor,
std::shared_ptr<crypto::Sr25519Provider> crypto_provider,
std::shared_ptr<network::PeerManager> pm,
std::shared_ptr<network::Router> router,
Expand Down Expand Up @@ -877,7 +877,7 @@ namespace kagome::parachain {
StorePair<Hash, DistribBlockEntry>>
store_;

std::shared_ptr<ParachainProcessorImpl> parachain_processor_;
std::shared_ptr<ParachainProcessor> parachain_processor_;
std::shared_ptr<crypto::Sr25519Provider> crypto_provider_;
std::shared_ptr<network::PeerManager> pm_;
std::shared_ptr<network::Router> router_;
Expand Down
Loading
Loading