Skip to content
This repository has been archived by the owner on Nov 15, 2023. It is now read-only.

Asynchronous backing PR #2300

Merged
merged 41 commits into from
Aug 18, 2023
Merged
Show file tree
Hide file tree
Changes from 37 commits
Commits
Show all changes
41 commits
Select commit Hold shift + click to select a range
35afcf7
Update substrate & polkadot
slumber Mar 9, 2023
6a32271
min changes to make async backing compile
slumber Mar 9, 2023
9f8e3fc
Merge remote-tracking branch 'origin/master' into slumber-async-backi…
slumber Apr 19, 2023
86b5d3d
(async backing) parachain-system: track limitations for unincluded bl…
slumber Apr 29, 2023
dcf62e5
refactor unincluded segment length into a ConsensusHook (#2501)
rphmeier May 17, 2023
da07585
parachain-system: ignore go ahead signal once upgrade is processed (#…
slumber May 22, 2023
59450a7
parachain-system: drop processed messages from inherent data (#2590)
slumber May 23, 2023
91f53dd
Merge remote-tracking branch 'origin/master' into slumber-async-backi…
slumber May 25, 2023
6141995
clippy
slumber May 25, 2023
f09c905
aura-ext: check slot in consensus hook and remove all `CheckInherents…
slumber Jul 3, 2023
d276682
Merge remote-tracking branch 'origin/master' into slumber-async-backi…
slumber Jul 5, 2023
3772536
update polkadot git refs
rphmeier Jul 10, 2023
bd68bf9
CollationGenerationConfig closure is now optional (#2772)
rphmeier Jul 11, 2023
99ec54f
Merge branch 'master' into slumber-async-backing-feature
rphmeier Jul 12, 2023
59987af
Merge branch 'master' into slumber-async-backing-feature
rphmeier Jul 13, 2023
3cb039b
propagate network-protocol-staging feature (#2899)
rphmeier Jul 19, 2023
51d9e9d
Feature Flagging Consensus Hook Type Parameter (#2911)
BradleyOlson64 Jul 20, 2023
532defc
Merge branch 'master' into slumber-async-backing-feature
rphmeier Jul 26, 2023
43f1835
fmt
rphmeier Jul 26, 2023
b9c1922
Merge branch 'master' into slumber-async-backing-feature
rphmeier Jul 28, 2023
e686dbb
bump deps and remove warning
rphmeier Jul 28, 2023
1937edb
parachain-system: update RelevantMessagingState according to the unin…
rphmeier Aug 2, 2023
fe3d6ee
Integrate new Aura / Parachain Consensus Logic in Parachain-Template …
rphmeier Aug 3, 2023
3dd4757
update comment
rphmeier Aug 7, 2023
84565ec
(async backing) restore `CheckInherents` for backwards-compatibility …
slumber Aug 8, 2023
107921d
Merge branch 'master' into slumber-async-backing-feature
rphmeier Aug 16, 2023
463d6e5
attempt
rphmeier Aug 16, 2023
385b818
properly end system blocks
rphmeier Aug 16, 2023
4b4e353
add some more comments
rphmeier Aug 17, 2023
17db1fb
ignore failing system parachain tests
rphmeier Aug 17, 2023
0897cad
Merge branch 'master' into slumber-async-backing-feature
rphmeier Aug 17, 2023
6c1bfa7
update refs after main feature branch merge
rphmeier Aug 17, 2023
694cd3a
comment out the offending tests because CI runs ignored tests
rphmeier Aug 17, 2023
aba9946
fix warnings
rphmeier Aug 18, 2023
1f53f90
fmt
rphmeier Aug 18, 2023
4f8910f
revert to polkadot master
slumber Aug 18, 2023
c9add91
Merge branch 'master' into slumber-async-backing-feature
eskimor Aug 18, 2023
cc66f0f
Merge remote-tracking branch 'origin/master' into slumber-async-backi…
slumber Aug 18, 2023
800512d
Merge remote-tracking branch 'origin/slumber-async-backing-feature' i…
slumber Aug 18, 2023
e2c2005
Merge branch 'master' into slumber-async-backing-feature
rphmeier Aug 18, 2023
5188e9c
cargo update -p polkadot-primitives -p sp-io
ordian Aug 18, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1,137 changes: 597 additions & 540 deletions Cargo.lock

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions bridges/bin/runtime-common/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,8 @@ sp-std = { git = "https://github.com/paritytech/substrate", branch = "master", d
sp-trie = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }

# Polkadot dependencies
xcm = { git = "https://github.com/paritytech/polkadot", branch = "master", default-features = false }
xcm-builder = { git = "https://github.com/paritytech/polkadot", branch = "master", default-features = false }
xcm = { git = "https://github.com/paritytech/polkadot", default-features = false , branch = "master" }
xcm-builder = { git = "https://github.com/paritytech/polkadot", default-features = false , branch = "master" }

[dev-dependencies]
bp-test-utils = { path = "../../primitives/test-utils" }
Expand Down
4 changes: 2 additions & 2 deletions bridges/modules/xcm-bridge-hub-router/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -26,8 +26,8 @@ sp-std = { git = "https://github.com/paritytech/substrate", branch = "master", d

# Polkadot Dependencies

xcm = { git = "https://github.com/paritytech/polkadot", branch = "master", default-features = false }
xcm-builder = { git = "https://github.com/paritytech/polkadot", branch = "master", default-features = false }
xcm = { git = "https://github.com/paritytech/polkadot", default-features = false , branch = "master" }
xcm-builder = { git = "https://github.com/paritytech/polkadot", default-features = false , branch = "master" }

[dev-dependencies]
sp-io = { git = "https://github.com/paritytech/substrate", branch = "master" }
Expand Down
2 changes: 1 addition & 1 deletion bridges/primitives/chain-bridge-hub-cumulus/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ sp-api = { git = "https://github.com/paritytech/substrate", branch = "master", d
sp-std = { git = "https://github.com/paritytech/substrate", branch = "master", default-features = false }

# Polkadot Dependencies
polkadot-primitives = { git = "https://github.com/paritytech/polkadot", branch = "master", default-features = false }
polkadot-primitives = { git = "https://github.com/paritytech/polkadot", default-features = false , branch = "master" }

[features]
default = ["std"]
Expand Down
1 change: 1 addition & 0 deletions client/collator/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -42,3 +42,4 @@ polkadot-node-subsystem-test-helpers = { git = "https://github.com/paritytech/po
# Cumulus
cumulus-test-client = { path = "../../test/client" }
cumulus-test-runtime = { path = "../../test/runtime" }
cumulus-test-relay-sproof-builder = { path = "../../test/relay-sproof-builder" }
72 changes: 64 additions & 8 deletions client/collator/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,8 @@ use sp_core::traits::SpawnNamed;
use sp_runtime::traits::{Block as BlockT, Header as HeaderT};

use cumulus_client_consensus_common::ParachainConsensus;
use polkadot_node_primitives::{CollationResult, MaybeCompressedPoV};
use polkadot_node_primitives::{CollationGenerationConfig, CollationResult, MaybeCompressedPoV};
use polkadot_node_subsystem::messages::{CollationGenerationMessage, CollatorProtocolMessage};
use polkadot_overseer::Handle as OverseerHandle;
use polkadot_primitives::{CollatorPair, Id as ParaId};

Expand Down Expand Up @@ -200,7 +201,7 @@ pub mod relay_chain_driven {
let config = CollationGenerationConfig {
key,
para_id,
collator: Box::new(move |relay_parent, validation_data| {
collator: Some(Box::new(move |relay_parent, validation_data| {
// Cloning the channel on each usage effectively makes the channel
// unbounded. The channel is actually bounded by the block production
// and consensus systems of Polkadot, which limits the amount of possible
Expand All @@ -218,7 +219,7 @@ pub mod relay_chain_driven {

this_rx.await.ok().flatten()
})
}),
})),
};

overseer_handle
Expand All @@ -233,6 +234,31 @@ pub mod relay_chain_driven {
}
}

/// Initialize the collation-related subsystems on the relay-chain side.
///
/// This must be done prior to collation, and does not set up any callback for collation.
/// For callback-driven collators, use the [`relay_chain_driven`] module.
pub async fn initialize_collator_subsystems(
overseer_handle: &mut OverseerHandle,
key: CollatorPair,
para_id: ParaId,
) {
overseer_handle
.send_msg(
CollationGenerationMessage::Initialize(CollationGenerationConfig {
key,
para_id,
collator: None,
}),
"StartCollator",
)
.await;

overseer_handle
.send_msg(CollatorProtocolMessage::CollateOn(para_id), "StartCollator")
.await;
}

/// Parameters for [`start_collator`].
pub struct StartCollatorParams<Block: BlockT, RA, BS, Spawner> {
pub para_id: ParaId,
Expand All @@ -246,7 +272,24 @@ pub struct StartCollatorParams<Block: BlockT, RA, BS, Spawner> {
}

/// Start the collator.
#[deprecated = "Collators should run consensus futures which handle this logic internally"]
pub async fn start_collator<Block, RA, BS, Spawner>(
params: StartCollatorParams<Block, RA, BS, Spawner>,
) where
Block: BlockT,
BS: BlockBackend<Block> + Send + Sync + 'static,
Spawner: SpawnNamed + Clone + Send + Sync + 'static,
RA: ProvideRuntimeApi<Block> + Send + Sync + 'static,
RA::Api: CollectCollationInfo<Block>,
{
// This never needed to be asynchronous, but shouldn't be changed due to backcompat.
#[allow(deprecated)]
start_collator_sync(params);
}

/// Start the collator in a synchronous function.
#[deprecated = "Collators should run consensus futures which handle this logic internally"]
pub fn start_collator_sync<Block, RA, BS, Spawner>(
StartCollatorParams {
para_id,
block_status,
Expand All @@ -269,9 +312,8 @@ pub async fn start_collator<Block, RA, BS, Spawner>(

let collator = Collator::new(collator_service, parachain_consensus);

let mut request_stream = relay_chain_driven::init(key, para_id, overseer_handle).await;

let collation_future = Box::pin(async move {
let mut request_stream = relay_chain_driven::init(key, para_id, overseer_handle).await;
while let Some(request) = request_stream.next().await {
let collation = collator
.clone()
Expand All @@ -298,11 +340,14 @@ mod tests {
Client, ClientBlockImportExt, DefaultTestClientBuilderExt, InitBlockBuilder,
TestClientBuilder, TestClientBuilderExt,
};
use cumulus_test_relay_sproof_builder::RelayStateSproofBuilder;
use cumulus_test_runtime::{Block, Header};
use futures::{channel::mpsc, executor::block_on, StreamExt};
use polkadot_node_primitives::CollationGenerationConfig;
use polkadot_node_subsystem::messages::CollationGenerationMessage;
use polkadot_node_subsystem_test_helpers::ForwardSubsystem;
use polkadot_overseer::{dummy::dummy_overseer_builder, HeadSupportsParachains};
use polkadot_primitives::HeadData;
use sp_consensus::BlockOrigin;
use sp_core::{testing::TaskExecutor, Pair};
use sp_runtime::traits::BlakeTwo256;
Expand Down Expand Up @@ -330,10 +375,14 @@ mod tests {
_: PHash,
validation_data: &PersistedValidationData,
) -> Option<ParachainCandidate<Block>> {
let mut sproof = RelayStateSproofBuilder::default();
sproof.included_para_head = Some(HeadData(parent.encode()));
sproof.para_id = cumulus_test_runtime::PARACHAIN_ID.into();

let builder = self.client.init_block_builder_at(
parent.hash(),
Some(validation_data.clone()),
Default::default(),
sproof,
);

let (block, _, proof) = builder.build().expect("Creates block").into_inner();
Expand Down Expand Up @@ -368,6 +417,7 @@ mod tests {

spawner.spawn("overseer", None, overseer.run().then(|_| async {}).boxed());

#[allow(deprecated)]
let collator_start = start_collator(StartCollatorParams {
runtime_api: client.clone(),
block_status: client.clone(),
Expand All @@ -384,13 +434,19 @@ mod tests {
.0
.expect("message should be send by `start_collator` above.");

let CollationGenerationMessage::Initialize(config) = msg;
let collator_fn = match msg {
CollationGenerationMessage::Initialize(CollationGenerationConfig {
collator: Some(c),
..
}) => c,
_ => panic!("unexpected message or no collator fn"),
};

let validation_data =
PersistedValidationData { parent_head: header.encode().into(), ..Default::default() };
let relay_parent = Default::default();

let collation = block_on((config.collator)(relay_parent, &validation_data))
let collation = block_on(collator_fn(relay_parent, &validation_data))
.expect("Collation is build")
.collation;

Expand Down
1 change: 1 addition & 0 deletions client/consensus/aura/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -45,4 +45,5 @@ cumulus-client-collator = { path = "../../collator" }
# Polkadot
polkadot-primitives = { git = "https://github.com/paritytech/polkadot", branch = "master" }
polkadot-node-primitives = { git = "https://github.com/paritytech/polkadot", branch = "master" }
polkadot-node-subsystem = { git = "https://github.com/paritytech/polkadot", branch = "master" }
polkadot-overseer = { git = "https://github.com/paritytech/polkadot", branch = "master" }
46 changes: 32 additions & 14 deletions client/consensus/aura/src/collator.rs
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,7 @@ use sp_runtime::{
};
use sp_state_machine::StorageChanges;
use sp_timestamp::Timestamp;
use std::{convert::TryFrom, error::Error, sync::Arc, time::Duration};
use std::{convert::TryFrom, error::Error, time::Duration};

/// Parameters for instantiating a [`Collator`].
pub struct Params<BI, CIDP, RClient, Proposer, CS> {
Expand All @@ -64,7 +64,7 @@ pub struct Params<BI, CIDP, RClient, Proposer, CS> {
/// The block import handle.
pub block_import: BI,
/// An interface to the relay-chain client.
pub relay_client: Arc<RClient>,
pub relay_client: RClient,
/// The keystore handle used for accessing parachain key material.
pub keystore: KeystorePtr,
/// The identifier of the parachain within the relay-chain.
Expand All @@ -81,12 +81,12 @@ pub struct Params<BI, CIDP, RClient, Proposer, CS> {
pub struct Collator<Block, P, BI, CIDP, RClient, Proposer, CS> {
create_inherent_data_providers: CIDP,
block_import: BI,
relay_client: Arc<RClient>,
relay_client: RClient,
keystore: KeystorePtr,
para_id: ParaId,
proposer: Proposer,
collator_service: CS,
_marker: std::marker::PhantomData<(Block, P)>,
_marker: std::marker::PhantomData<(Block, Box<dyn Fn(P) + Send + Sync + 'static>)>,
}

impl<Block, P, BI, CIDP, RClient, Proposer, CS> Collator<Block, P, BI, CIDP, RClient, Proposer, CS>
Expand Down Expand Up @@ -123,7 +123,7 @@ where
validation_data: &PersistedValidationData,
parent_hash: Block::Hash,
timestamp: impl Into<Option<Timestamp>>,
) -> Result<(ParachainInherentData, InherentData), Box<dyn Error>> {
) -> Result<(ParachainInherentData, InherentData), Box<dyn Error + Send + Sync + 'static>> {
let paras_inherent_data = ParachainInherentData::create_at(
relay_parent,
&self.relay_client,
Expand All @@ -143,7 +143,7 @@ where
let mut other_inherent_data = self
.create_inherent_data_providers
.create_inherent_data_providers(parent_hash, ())
.map_err(|e| e as Box<dyn Error>)
.map_err(|e| e as Box<dyn Error + Send + Sync + 'static>)
.await?
.create_inherent_data()
.await
Expand Down Expand Up @@ -172,7 +172,8 @@ where
inherent_data: (ParachainInherentData, InherentData),
proposal_duration: Duration,
max_pov_size: usize,
) -> Result<(Collation, ParachainBlockData<Block>, Block::Hash), Box<dyn Error>> {
) -> Result<(Collation, ParachainBlockData<Block>, Block::Hash), Box<dyn Error + Send + 'static>>
{
let mut digest = additional_pre_digest.into().unwrap_or_default();
digest.push(slot_claim.pre_digest.clone());

Expand All @@ -187,14 +188,15 @@ where
Some(max_pov_size),
)
.await
.map_err(|e| Box::new(e))?;
.map_err(|e| Box::new(e) as Box<dyn Error + Send>)?;

let sealed_importable = seal::<_, P>(
proposal.block,
proposal.storage_changes,
&slot_claim.author_pub,
&self.keystore,
)?;
)
.map_err(|e| e as Box<dyn Error + Send>)?;

let post_hash = sealed_importable.post_hash();
let block = Block::new(
Expand All @@ -206,7 +208,10 @@ where
.clone(),
);

self.block_import.import_block(sealed_importable).await?;
self.block_import
.import_block(sealed_importable)
.map_err(|e| Box::new(e) as Box<dyn Error + Send>)
.await?;

if let Some((collation, block_data)) = self.collator_service.build_collation(
parent_header,
Expand All @@ -231,7 +236,8 @@ where

Ok((collation, block_data, post_hash))
} else {
Err("Unable to produce collation".to_string().into())
Err(Box::<dyn Error + Send + Sync>::from("Unable to produce collation")
as Box<dyn Error + Send>)
}
}

Expand Down Expand Up @@ -285,7 +291,7 @@ pub async fn claim_slot<B, C, P>(
parent_hash: B::Hash,
relay_parent_header: &PHeader,
slot_duration: SlotDuration,
relay_chain_slot_duration: SlotDuration,
relay_chain_slot_duration: Duration,
keystore: &KeystorePtr,
) -> Result<Option<SlotClaim<P::Public>>, Box<dyn Error>>
where
Expand All @@ -304,7 +310,19 @@ where
relay_parent_header,
relay_chain_slot_duration,
) {
Some((_, t)) => (Slot::from_timestamp(t, slot_duration), t),
Some((r_s, t)) => {
let our_slot = Slot::from_timestamp(t, slot_duration);
tracing::debug!(
target: crate::LOG_TARGET,
relay_slot = ?r_s,
para_slot = ?our_slot,
timestamp = ?t,
?slot_duration,
?relay_chain_slot_duration,
"Adjusted relay-chain slot to parachain slot"
);
(our_slot, t)
},
None => return Ok(None),
};

Expand All @@ -326,7 +344,7 @@ pub fn seal<B: BlockT, P>(
storage_changes: StorageChanges<HashingFor<B>>,
author_pub: &P::Public,
keystore: &KeystorePtr,
) -> Result<BlockImportParams<B>, Box<dyn Error>>
) -> Result<BlockImportParams<B>, Box<dyn Error + Send + Sync + 'static>>
where
P: Pair,
P::Signature: Codec + TryFrom<Vec<u8>>,
Expand Down
Loading