From ff40792855dda24a00ffb1b87a0853d2af5b1943 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Wed, 19 Jun 2024 16:48:03 +0200 Subject: [PATCH 01/48] Add execution priority --- polkadot/node/core/pvf/src/execute/queue.rs | 34 ++++++++++++++++----- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/polkadot/node/core/pvf/src/execute/queue.rs b/polkadot/node/core/pvf/src/execute/queue.rs index bb00a5a652d64..2d315cc494b18 100644 --- a/polkadot/node/core/pvf/src/execute/queue.rs +++ b/polkadot/node/core/pvf/src/execute/queue.rs @@ -37,7 +37,7 @@ use polkadot_node_core_pvf_common::{ use polkadot_primitives::{ExecutorParams, ExecutorParamsHash}; use slotmap::HopSlotMap; use std::{ - collections::VecDeque, + collections::{HashMap, VecDeque}, fmt, path::PathBuf, time::{Duration, Instant}, @@ -57,6 +57,12 @@ pub enum ToQueue { Enqueue { artifact: ArtifactPathId, pending_execution_request: PendingExecutionRequest }, } +#[derive(Debug, Hash, PartialEq, Eq)] +pub enum ExecutePriority { + Normal, + Critical, +} + /// A response from queue. #[derive(Debug)] pub enum FromQueue { @@ -162,7 +168,7 @@ struct Queue { security_status: SecurityStatus, /// The queue of jobs that are waiting for a worker to pick up. - queue: VecDeque, + unscheduled: HashMap>, workers: Workers, mux: Mux, } @@ -188,7 +194,7 @@ impl Queue { security_status, to_queue_rx, from_queue_tx, - queue: VecDeque::new(), + unscheduled: HashMap::new(), mux: Mux::new(), workers: Workers { running: HopSlotMap::with_capacity_and_key(10), @@ -224,7 +230,18 @@ impl Queue { fn try_assign_next_job(&mut self, finished_worker: Option) { // New jobs are always pushed to the tail of the queue; the one at its head is always // the eldest one. - let eldest = if let Some(eldest) = self.queue.get(0) { eldest } else { return }; + + let has_critical = self + .unscheduled + .get(&ExecutePriority::Critical) + .map_or(false, |v| !v.is_empty()); + let queue_priority = has_critical + .then(|| ExecutePriority::Critical) + .unwrap_or(ExecutePriority::Normal); + + let Some(queue) = self.unscheduled.get_mut(&queue_priority) else { return }; + + let eldest = if let Some(eldest) = queue.get(0) { eldest } else { return }; // By default, we're going to execute the eldest job on any worker slot available, even if // we have to kill and re-spawn a worker @@ -236,7 +253,7 @@ impl Queue { if eldest.waiting_since.elapsed() < MAX_KEEP_WAITING { if let Some(finished_worker) = finished_worker { if let Some(worker_data) = self.workers.running.get(finished_worker) { - for (i, job) in self.queue.iter().enumerate() { + for (i, job) in queue.iter().enumerate() { if worker_data.executor_params_hash == job.executor_params.hash() { (worker, job_index) = (Some(finished_worker), i); break @@ -248,7 +265,7 @@ impl Queue { if worker.is_none() { // Try to obtain a worker for the job - worker = self.workers.find_available(self.queue[job_index].executor_params.hash()); + worker = self.workers.find_available(queue[job_index].executor_params.hash()); } if worker.is_none() { @@ -266,7 +283,7 @@ impl Queue { return } - let job = self.queue.remove(job_index).expect("Job is just checked to be in queue; qed"); + let job = queue.remove(job_index).expect("Job is just checked to be in queue; qed"); if let Some(worker) = worker { assign(self, worker, job); @@ -292,6 +309,7 @@ async fn purge_dead(metrics: &Metrics, workers: &mut Workers) { } fn handle_to_queue(queue: &mut Queue, to_queue: ToQueue) { + let priority = ExecutePriority::Normal; let ToQueue::Enqueue { artifact, pending_execution_request } = to_queue; let PendingExecutionRequest { exec_timeout, params, executor_params, result_tx } = pending_execution_request; @@ -309,7 +327,7 @@ fn handle_to_queue(queue: &mut Queue, to_queue: ToQueue) { result_tx, waiting_since: Instant::now(), }; - queue.queue.push_back(job); + queue.unscheduled.entry(priority).or_default().push_back(job); queue.try_assign_next_job(None); } From fca9a11ce2eeee70ed4d28491ddcce4895278528 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Wed, 19 Jun 2024 17:16:33 +0200 Subject: [PATCH 02/48] Move priority to PendingExecutionRequest --- polkadot/node/core/pvf/src/execute/queue.rs | 21 +++++---- polkadot/node/core/pvf/src/host.rs | 49 ++++++++++++++++++--- polkadot/node/core/pvf/src/lib.rs | 2 +- polkadot/node/core/pvf/src/priority.rs | 6 +++ polkadot/node/core/pvf/tests/it/main.rs | 1 + 5 files changed, 61 insertions(+), 18 deletions(-) diff --git a/polkadot/node/core/pvf/src/execute/queue.rs b/polkadot/node/core/pvf/src/execute/queue.rs index 2d315cc494b18..0cc9de7698563 100644 --- a/polkadot/node/core/pvf/src/execute/queue.rs +++ b/polkadot/node/core/pvf/src/execute/queue.rs @@ -22,7 +22,7 @@ use crate::{ host::ResultSender, metrics::Metrics, worker_interface::{IdleWorker, WorkerHandle}, - InvalidCandidate, PossiblyInvalidError, ValidationError, LOG_TARGET, + ExecutePriority, InvalidCandidate, PossiblyInvalidError, ValidationError, LOG_TARGET, }; use futures::{ channel::{mpsc, oneshot}, @@ -57,12 +57,6 @@ pub enum ToQueue { Enqueue { artifact: ArtifactPathId, pending_execution_request: PendingExecutionRequest }, } -#[derive(Debug, Hash, PartialEq, Eq)] -pub enum ExecutePriority { - Normal, - Critical, -} - /// A response from queue. #[derive(Debug)] pub enum FromQueue { @@ -77,6 +71,7 @@ pub struct PendingExecutionRequest { pub params: Vec, pub executor_params: ExecutorParams, pub result_tx: ResultSender, + pub execute_priority: ExecutePriority, } struct ExecuteJob { @@ -309,10 +304,14 @@ async fn purge_dead(metrics: &Metrics, workers: &mut Workers) { } fn handle_to_queue(queue: &mut Queue, to_queue: ToQueue) { - let priority = ExecutePriority::Normal; let ToQueue::Enqueue { artifact, pending_execution_request } = to_queue; - let PendingExecutionRequest { exec_timeout, params, executor_params, result_tx } = - pending_execution_request; + let PendingExecutionRequest { + exec_timeout, + params, + executor_params, + result_tx, + execute_priority: execution_priority, + } = pending_execution_request; gum::debug!( target: LOG_TARGET, validation_code_hash = ?artifact.id.code_hash, @@ -327,7 +326,7 @@ fn handle_to_queue(queue: &mut Queue, to_queue: ToQueue) { result_tx, waiting_since: Instant::now(), }; - queue.unscheduled.entry(priority).or_default().push_back(job); + queue.unscheduled.entry(execution_priority).or_default().push_back(job); queue.try_assign_next_job(None); } diff --git a/polkadot/node/core/pvf/src/host.rs b/polkadot/node/core/pvf/src/host.rs index 462631d33b525..9074b963786c0 100644 --- a/polkadot/node/core/pvf/src/host.rs +++ b/polkadot/node/core/pvf/src/host.rs @@ -24,7 +24,7 @@ use crate::{ artifacts::{ArtifactId, ArtifactPathId, ArtifactState, Artifacts, ArtifactsCleanupConfig}, execute::{self, PendingExecutionRequest}, metrics::Metrics, - prepare, Priority, SecurityStatus, ValidationError, LOG_TARGET, + prepare, ExecutePriority, Priority, SecurityStatus, ValidationError, LOG_TARGET, }; use always_assert::never; use futures::{ @@ -110,6 +110,7 @@ impl ValidationHost { exec_timeout: Duration, params: Vec, priority: Priority, + execute_priority: ExecutePriority, result_tx: ResultSender, ) -> Result<(), String> { self.to_host_tx @@ -118,6 +119,7 @@ impl ValidationHost { exec_timeout, params, priority, + execute_priority, result_tx, })) .await @@ -149,6 +151,7 @@ struct ExecutePvfInputs { exec_timeout: Duration, params: Vec, priority: Priority, + execute_priority: ExecutePriority, result_tx: ResultSender, } @@ -539,7 +542,8 @@ async fn handle_execute_pvf( awaiting_prepare: &mut AwaitingPrepare, inputs: ExecutePvfInputs, ) -> Result<(), Fatal> { - let ExecutePvfInputs { pvf, exec_timeout, params, priority, result_tx } = inputs; + let ExecutePvfInputs { pvf, exec_timeout, params, priority, execute_priority, result_tx } = + inputs; let artifact_id = ArtifactId::from_pvf_prep_data(&pvf); let executor_params = (*pvf.executor_params()).clone(); @@ -560,6 +564,7 @@ async fn handle_execute_pvf( exec_timeout, params, executor_params, + execute_priority, result_tx, }, }, @@ -589,6 +594,7 @@ async fn handle_execute_pvf( exec_timeout, params, executor_params, + execute_priority, result_tx, }, ) @@ -598,7 +604,13 @@ async fn handle_execute_pvf( ArtifactState::Preparing { .. } => { awaiting_prepare.add( artifact_id, - PendingExecutionRequest { exec_timeout, params, executor_params, result_tx }, + PendingExecutionRequest { + exec_timeout, + params, + executor_params, + result_tx, + execute_priority, + }, ); }, ArtifactState::FailedToProcess { last_time_failed, num_failures, error } => { @@ -629,6 +641,7 @@ async fn handle_execute_pvf( exec_timeout, params, executor_params, + execute_priority, result_tx, }, ) @@ -648,7 +661,13 @@ async fn handle_execute_pvf( pvf, priority, artifact_id, - PendingExecutionRequest { exec_timeout, params, executor_params, result_tx }, + PendingExecutionRequest { + exec_timeout, + params, + executor_params, + result_tx, + execute_priority, + }, ) .await?; } @@ -770,8 +789,13 @@ async fn handle_prepare_done( // It's finally time to dispatch all the execution requests that were waiting for this artifact // to be prepared. let pending_requests = awaiting_prepare.take(&artifact_id); - for PendingExecutionRequest { exec_timeout, params, executor_params, result_tx } in - pending_requests + for PendingExecutionRequest { + exec_timeout, + params, + executor_params, + result_tx, + execute_priority, + } in pending_requests { if result_tx.is_canceled() { // Preparation could've taken quite a bit of time and the requester may be not @@ -795,6 +819,7 @@ async fn handle_prepare_done( exec_timeout, params, executor_params, + execute_priority, result_tx, }, }, @@ -1230,6 +1255,7 @@ pub(crate) mod tests { TEST_EXECUTION_TIMEOUT, b"pvf1".to_vec(), Priority::Normal, + ExecutePriority::Normal, result_tx, ) .await @@ -1241,6 +1267,7 @@ pub(crate) mod tests { TEST_EXECUTION_TIMEOUT, b"pvf1".to_vec(), Priority::Critical, + ExecutePriority::Normal, result_tx, ) .await @@ -1252,6 +1279,7 @@ pub(crate) mod tests { TEST_EXECUTION_TIMEOUT, b"pvf2".to_vec(), Priority::Normal, + ExecutePriority::Normal, result_tx, ) .await @@ -1393,6 +1421,7 @@ pub(crate) mod tests { TEST_EXECUTION_TIMEOUT, b"pvf2".to_vec(), Priority::Critical, + ExecutePriority::Normal, result_tx, ) .await @@ -1440,6 +1469,7 @@ pub(crate) mod tests { TEST_EXECUTION_TIMEOUT, b"pvf2".to_vec(), Priority::Critical, + ExecutePriority::Normal, result_tx, ) .await @@ -1542,6 +1572,7 @@ pub(crate) mod tests { TEST_EXECUTION_TIMEOUT, b"pvf".to_vec(), Priority::Critical, + ExecutePriority::Normal, result_tx, ) .await @@ -1572,6 +1603,7 @@ pub(crate) mod tests { TEST_EXECUTION_TIMEOUT, b"pvf".to_vec(), Priority::Critical, + ExecutePriority::Normal, result_tx_2, ) .await @@ -1594,6 +1626,7 @@ pub(crate) mod tests { TEST_EXECUTION_TIMEOUT, b"pvf".to_vec(), Priority::Critical, + ExecutePriority::Normal, result_tx_3, ) .await @@ -1644,6 +1677,7 @@ pub(crate) mod tests { TEST_EXECUTION_TIMEOUT, b"pvf".to_vec(), Priority::Critical, + ExecutePriority::Normal, result_tx, ) .await @@ -1674,6 +1708,7 @@ pub(crate) mod tests { TEST_EXECUTION_TIMEOUT, b"pvf".to_vec(), Priority::Critical, + ExecutePriority::Normal, result_tx_2, ) .await @@ -1696,6 +1731,7 @@ pub(crate) mod tests { TEST_EXECUTION_TIMEOUT, b"pvf".to_vec(), Priority::Critical, + ExecutePriority::Normal, result_tx_3, ) .await @@ -1762,6 +1798,7 @@ pub(crate) mod tests { TEST_EXECUTION_TIMEOUT, b"pvf1".to_vec(), Priority::Normal, + ExecutePriority::Normal, result_tx, ) .await diff --git a/polkadot/node/core/pvf/src/lib.rs b/polkadot/node/core/pvf/src/lib.rs index 462498fa8f6b1..f5b3aa7b627a4 100644 --- a/polkadot/node/core/pvf/src/lib.rs +++ b/polkadot/node/core/pvf/src/lib.rs @@ -110,7 +110,7 @@ pub use host::{ PREPARE_BINARY_NAME, }; pub use metrics::Metrics; -pub use priority::Priority; +pub use priority::{ExecutePriority, Priority}; pub use worker_interface::{framed_recv, framed_send, JOB_TIMEOUT_WALL_CLOCK_FACTOR}; // Re-export some common types. diff --git a/polkadot/node/core/pvf/src/priority.rs b/polkadot/node/core/pvf/src/priority.rs index 0d18d4b484cab..0b20d91ec115f 100644 --- a/polkadot/node/core/pvf/src/priority.rs +++ b/polkadot/node/core/pvf/src/priority.rs @@ -35,3 +35,9 @@ impl Priority { self == Priority::Critical } } + +#[derive(Debug, Hash, PartialEq, Eq, Clone, Copy)] +pub enum ExecutePriority { + Normal, + Critical, +} diff --git a/polkadot/node/core/pvf/tests/it/main.rs b/polkadot/node/core/pvf/tests/it/main.rs index d62a1aef23099..0b5064ee75c32 100644 --- a/polkadot/node/core/pvf/tests/it/main.rs +++ b/polkadot/node/core/pvf/tests/it/main.rs @@ -124,6 +124,7 @@ impl TestHost { TEST_EXECUTION_TIMEOUT, params.encode(), polkadot_node_core_pvf::Priority::Normal, + polkadot_node_core_pvf::ExecutePriority::Normal, result_tx, ) .await From 7ec65f8aa6197871092da4c497f9d793a85d96ea Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Wed, 19 Jun 2024 17:20:58 +0200 Subject: [PATCH 03/48] Update candidate validation --- .../node/core/candidate-validation/src/lib.rs | 29 ++++++++++++++++--- .../core/candidate-validation/src/tests.rs | 2 ++ polkadot/node/core/pvf/src/priority.rs | 3 ++ 3 files changed, 30 insertions(+), 4 deletions(-) diff --git a/polkadot/node/core/candidate-validation/src/lib.rs b/polkadot/node/core/candidate-validation/src/lib.rs index 76619bd391f2b..2dfc6fe4894cb 100644 --- a/polkadot/node/core/candidate-validation/src/lib.rs +++ b/polkadot/node/core/candidate-validation/src/lib.rs @@ -676,6 +676,7 @@ async fn validate_candidate_exhaustive( exec_timeout, params.encode(), polkadot_node_core_pvf::Priority::Normal, + polkadot_node_core_pvf::ExecutePriority::Normal, ) .await }, @@ -688,6 +689,7 @@ async fn validate_candidate_exhaustive( executor_params, PVF_APPROVAL_EXECUTION_RETRY_DELAY, polkadot_node_core_pvf::Priority::Critical, + polkadot_node_core_pvf::ExecutePriority::Normal, ) .await, }; @@ -772,6 +774,8 @@ trait ValidationBackend { encoded_params: Vec, // The priority for the preparation job. prepare_priority: polkadot_node_core_pvf::Priority, + // The priority for the preparation job. + execute_priority: polkadot_node_core_pvf::ExecutePriority, ) -> Result; /// Tries executing a PVF. Will retry once if an error is encountered that may have @@ -791,6 +795,8 @@ trait ValidationBackend { retry_delay: Duration, // The priority for the preparation job. prepare_priority: polkadot_node_core_pvf::Priority, + // The priority for the preparation job. + execute_priority: polkadot_node_core_pvf::ExecutePriority, ) -> Result { let prep_timeout = pvf_prep_timeout(&executor_params, PvfPrepKind::Prepare); // Construct the PVF a single time, since it is an expensive operation. Cloning it is cheap. @@ -806,7 +812,13 @@ trait ValidationBackend { // Use `Priority::Critical` as finality trumps parachain liveliness. let mut validation_result = self - .validate_candidate(pvf.clone(), exec_timeout, params.encode(), prepare_priority) + .validate_candidate( + pvf.clone(), + exec_timeout, + params.encode(), + prepare_priority, + execute_priority, + ) .await; if validation_result.is_ok() { return validation_result @@ -882,7 +894,13 @@ trait ValidationBackend { // Encode the params again when re-trying. We expect the retry case to be relatively // rare, and we want to avoid unconditionally cloning data. validation_result = self - .validate_candidate(pvf.clone(), new_timeout, params.encode(), prepare_priority) + .validate_candidate( + pvf.clone(), + new_timeout, + params.encode(), + prepare_priority, + execute_priority, + ) .await; } } @@ -903,10 +921,13 @@ impl ValidationBackend for ValidationHost { encoded_params: Vec, // The priority for the preparation job. prepare_priority: polkadot_node_core_pvf::Priority, + // The priority for the preparation job. + execute_priority: polkadot_node_core_pvf::ExecutePriority, ) -> Result { let (tx, rx) = oneshot::channel(); - if let Err(err) = - self.execute_pvf(pvf, exec_timeout, encoded_params, prepare_priority, tx).await + if let Err(err) = self + .execute_pvf(pvf, exec_timeout, encoded_params, prepare_priority, execute_priority, tx) + .await { return Err(InternalValidationError::HostCommunication(format!( "cannot send pvf to the validation host, it might have shut down: {:?}", diff --git a/polkadot/node/core/candidate-validation/src/tests.rs b/polkadot/node/core/candidate-validation/src/tests.rs index 491ed7a335d8f..cf36076cf5089 100644 --- a/polkadot/node/core/candidate-validation/src/tests.rs +++ b/polkadot/node/core/candidate-validation/src/tests.rs @@ -378,6 +378,7 @@ impl ValidationBackend for MockValidateCandidateBackend { _timeout: Duration, _encoded_params: Vec, _prepare_priority: polkadot_node_core_pvf::Priority, + _execute_priority: polkadot_node_core_pvf::ExecutePriority, ) -> Result { // This is expected to panic if called more times than expected, indicating an error in the // test. @@ -1064,6 +1065,7 @@ impl ValidationBackend for MockPreCheckBackend { _timeout: Duration, _encoded_params: Vec, _prepare_priority: polkadot_node_core_pvf::Priority, + _execute_priority: polkadot_node_core_pvf::ExecutePriority, ) -> Result { unreachable!() } diff --git a/polkadot/node/core/pvf/src/priority.rs b/polkadot/node/core/pvf/src/priority.rs index 0b20d91ec115f..2fab22dd4026d 100644 --- a/polkadot/node/core/pvf/src/priority.rs +++ b/polkadot/node/core/pvf/src/priority.rs @@ -36,8 +36,11 @@ impl Priority { } } +/// A priority assigned to execution of a PVF. #[derive(Debug, Hash, PartialEq, Eq, Clone, Copy)] pub enum ExecutePriority { + /// Normall Normal, + /// Critical Critical, } From 0d0926a7a101300129308d71df15c4f6141fa15a Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 20 Jun 2024 12:08:14 +0200 Subject: [PATCH 04/48] Add scheduling logic --- polkadot/node/core/pvf/src/execute/queue.rs | 167 ++++++++++++++++++-- polkadot/node/core/pvf/src/priority.rs | 26 ++- 2 files changed, 179 insertions(+), 14 deletions(-) diff --git a/polkadot/node/core/pvf/src/execute/queue.rs b/polkadot/node/core/pvf/src/execute/queue.rs index 0cc9de7698563..3b9cd9959da61 100644 --- a/polkadot/node/core/pvf/src/execute/queue.rs +++ b/polkadot/node/core/pvf/src/execute/queue.rs @@ -163,7 +163,7 @@ struct Queue { security_status: SecurityStatus, /// The queue of jobs that are waiting for a worker to pick up. - unscheduled: HashMap>, + unscheduled: Unscheduled, workers: Workers, mux: Mux, } @@ -189,7 +189,7 @@ impl Queue { security_status, to_queue_rx, from_queue_tx, - unscheduled: HashMap::new(), + unscheduled: Unscheduled::new(), mux: Mux::new(), workers: Workers { running: HopSlotMap::with_capacity_and_key(10), @@ -226,15 +226,7 @@ impl Queue { // New jobs are always pushed to the tail of the queue; the one at its head is always // the eldest one. - let has_critical = self - .unscheduled - .get(&ExecutePriority::Critical) - .map_or(false, |v| !v.is_empty()); - let queue_priority = has_critical - .then(|| ExecutePriority::Critical) - .unwrap_or(ExecutePriority::Normal); - - let Some(queue) = self.unscheduled.get_mut(&queue_priority) else { return }; + let Some((queue, priority)) = self.unscheduled.get_mut() else { return }; let eldest = if let Some(eldest) = queue.get(0) { eldest } else { return }; @@ -285,6 +277,7 @@ impl Queue { } else { spawn_extra_worker(self, job); } + self.unscheduled.log_priority(priority); } } @@ -310,7 +303,7 @@ fn handle_to_queue(queue: &mut Queue, to_queue: ToQueue) { params, executor_params, result_tx, - execute_priority: execution_priority, + execute_priority, } = pending_execution_request; gum::debug!( target: LOG_TARGET, @@ -326,7 +319,7 @@ fn handle_to_queue(queue: &mut Queue, to_queue: ToQueue) { result_tx, waiting_since: Instant::now(), }; - queue.unscheduled.entry(execution_priority).or_default().push_back(job); + queue.unscheduled.add(job, execute_priority); queue.try_assign_next_job(None); } @@ -625,3 +618,151 @@ pub fn start( .run(); (to_queue_tx, from_queue_rx, run) } + +struct Unscheduled { + unscheduled: HashMap>, + priority_log: VecDeque, +} + +impl Unscheduled { + /// The minimum number of items with normal priority that must be present, + /// regardless of the amount of critical priority items. + /// In percents. + const MIN_NORMAL_PRIORITY_PRESENCE: usize = 10; + + fn new() -> Self { + Self { + unscheduled: ExecutePriority::iter() + .map(|priority| (priority, VecDeque::new())) + .collect(), + priority_log: VecDeque::with_capacity(10), + } + } + + fn log_priority(&mut self, priority: ExecutePriority) { + if self.priority_log.len() == self.priority_log.capacity() { + let _ = self.priority_log.pop_front(); + } + self.priority_log.push_back(priority); + } + + fn select_next_priority(&self) -> ExecutePriority { + let normal_count = self.priority_log.iter().filter(|v| v.is_normal()).count(); + let normal_presense = if self.priority_log.len() == 0 { + 100 // Do critical first + } else { + normal_count * 100 / self.priority_log.len() + }; + let mut priority = if normal_presense < Self::MIN_NORMAL_PRIORITY_PRESENCE { + ExecutePriority::Normal + } else { + ExecutePriority::Critical + }; + + while self.unscheduled.get(&priority).map_or(true, |v| v.is_empty()) { + let Some(lower) = priority.lower() else { break }; + priority = lower; + } + + priority + } + + fn get_mut(&mut self) -> Option<(&mut VecDeque, ExecutePriority)> { + let priority = self.select_next_priority(); + self.unscheduled.get_mut(&priority).map(|jobs| (jobs, priority)) + } + + fn add(&mut self, job: ExecuteJob, priority: ExecutePriority) { + self.unscheduled.entry(priority).or_default().push_back(job); + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::testing::artifact_id; + use std::time::Duration; + + fn create_execution_job() -> ExecuteJob { + let (result_tx, _result_rx) = oneshot::channel(); + ExecuteJob { + artifact: ArtifactPathId { id: artifact_id(0), path: PathBuf::new() }, + exec_timeout: Duration::from_secs(10), + params: vec![], + executor_params: ExecutorParams::default(), + result_tx, + waiting_since: Instant::now(), + } + } + + #[test] + fn test_unscheduled_add() { + let mut unscheduled = Unscheduled::new(); + + unscheduled.add(create_execution_job(), ExecutePriority::Low); + unscheduled.add(create_execution_job(), ExecutePriority::Normal); + unscheduled.add(create_execution_job(), ExecutePriority::Critical); + + let low = unscheduled.unscheduled.get(&ExecutePriority::Low).unwrap(); + let normal = unscheduled.unscheduled.get(&ExecutePriority::Normal).unwrap(); + let critical = unscheduled.unscheduled.get(&ExecutePriority::Critical).unwrap(); + + assert_eq!(low.len(), 1); + assert_eq!(normal.len(), 1); + assert_eq!(critical.len(), 1); + } + + #[test] + fn test_log_priority() { + let mut unscheduled = Unscheduled::new(); + + for _ in 0..15 { + unscheduled.log_priority(ExecutePriority::Normal); + } + + assert_eq!(unscheduled.priority_log.len(), 10); + assert!(unscheduled.priority_log.iter().all(|&p| p == ExecutePriority::Normal)); + } + + #[test] + fn test_unscheduled_priority_selection() { + let mut unscheduled = Unscheduled::new(); + + unscheduled.add(create_execution_job(), ExecutePriority::Low); + + let (queue, priority) = unscheduled.get_mut().unwrap(); + + // Returns low priority queue if no others + assert_eq!(priority, ExecutePriority::Low); + assert_eq!(queue.len(), 1); + + for _ in 0..2 { + unscheduled.add(create_execution_job(), ExecutePriority::Normal); + } + + let (queue, priority) = unscheduled.get_mut().unwrap(); + + // Returns normal priority queue + assert_eq!(priority, ExecutePriority::Normal); + assert_eq!(queue.len(), 2); + + for _ in 0..3 { + unscheduled.add(create_execution_job(), ExecutePriority::Critical); + } + for _ in 0..10 { + unscheduled.log_priority(ExecutePriority::Critical); + } + let (queue, priority) = unscheduled.get_mut().unwrap(); + + // Returns normal priority queue because logged too many criticals + assert_eq!(priority, ExecutePriority::Normal); + assert_eq!(queue.len(), 2); + + unscheduled.log_priority(ExecutePriority::Normal); + let (queue, priority) = unscheduled.get_mut().unwrap(); + + // Returns critical priority queue because logged enough normals + assert_eq!(priority, ExecutePriority::Critical); + assert_eq!(queue.len(), 3); + } +} diff --git a/polkadot/node/core/pvf/src/priority.rs b/polkadot/node/core/pvf/src/priority.rs index 2fab22dd4026d..d504b036580bf 100644 --- a/polkadot/node/core/pvf/src/priority.rs +++ b/polkadot/node/core/pvf/src/priority.rs @@ -37,10 +37,34 @@ impl Priority { } /// A priority assigned to execution of a PVF. -#[derive(Debug, Hash, PartialEq, Eq, Clone, Copy)] +#[derive(Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Clone, Copy)] pub enum ExecutePriority { + /// Low + Low, /// Normall Normal, /// Critical Critical, } + +impl ExecutePriority { + /// Returns an iterator over the variants of `ExecutePriority` in order from `Low` to + /// `Critical`. + pub fn iter() -> impl Iterator { + [Self::Low, Self::Normal, Self::Critical].iter().copied() + } + + /// Returns the next lower priority level, or `None` if `self` is `Low`. + pub fn lower(&self) -> Option { + match self { + Self::Critical => Some(Self::Normal), + Self::Normal => Some(Self::Low), + Self::Low => None, + } + } + + /// Returns `true` if `self` is `Normal` + pub fn is_normal(&self) -> bool { + *self == Self::Normal + } +} From f9bf22546492f2f0bef12253442e2fbc4d5ac6e4 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 20 Jun 2024 12:10:12 +0200 Subject: [PATCH 05/48] Rename PreparePriority --- .../node/core/candidate-validation/src/lib.rs | 10 ++--- .../core/candidate-validation/src/tests.rs | 4 +- polkadot/node/core/pvf/src/host.rs | 43 ++++++++++--------- polkadot/node/core/pvf/src/lib.rs | 2 +- polkadot/node/core/pvf/src/prepare/queue.rs | 40 ++++++++--------- polkadot/node/core/pvf/src/priority.rs | 6 +-- polkadot/node/core/pvf/tests/it/main.rs | 2 +- 7 files changed, 55 insertions(+), 52 deletions(-) diff --git a/polkadot/node/core/candidate-validation/src/lib.rs b/polkadot/node/core/candidate-validation/src/lib.rs index 2dfc6fe4894cb..242ab5f1b51bf 100644 --- a/polkadot/node/core/candidate-validation/src/lib.rs +++ b/polkadot/node/core/candidate-validation/src/lib.rs @@ -675,7 +675,7 @@ async fn validate_candidate_exhaustive( pvf, exec_timeout, params.encode(), - polkadot_node_core_pvf::Priority::Normal, + polkadot_node_core_pvf::PreparePriority::Normal, polkadot_node_core_pvf::ExecutePriority::Normal, ) .await @@ -688,7 +688,7 @@ async fn validate_candidate_exhaustive( params, executor_params, PVF_APPROVAL_EXECUTION_RETRY_DELAY, - polkadot_node_core_pvf::Priority::Critical, + polkadot_node_core_pvf::PreparePriority::Critical, polkadot_node_core_pvf::ExecutePriority::Normal, ) .await, @@ -773,7 +773,7 @@ trait ValidationBackend { exec_timeout: Duration, encoded_params: Vec, // The priority for the preparation job. - prepare_priority: polkadot_node_core_pvf::Priority, + prepare_priority: polkadot_node_core_pvf::PreparePriority, // The priority for the preparation job. execute_priority: polkadot_node_core_pvf::ExecutePriority, ) -> Result; @@ -794,7 +794,7 @@ trait ValidationBackend { executor_params: ExecutorParams, retry_delay: Duration, // The priority for the preparation job. - prepare_priority: polkadot_node_core_pvf::Priority, + prepare_priority: polkadot_node_core_pvf::PreparePriority, // The priority for the preparation job. execute_priority: polkadot_node_core_pvf::ExecutePriority, ) -> Result { @@ -920,7 +920,7 @@ impl ValidationBackend for ValidationHost { exec_timeout: Duration, encoded_params: Vec, // The priority for the preparation job. - prepare_priority: polkadot_node_core_pvf::Priority, + prepare_priority: polkadot_node_core_pvf::PreparePriority, // The priority for the preparation job. execute_priority: polkadot_node_core_pvf::ExecutePriority, ) -> Result { diff --git a/polkadot/node/core/candidate-validation/src/tests.rs b/polkadot/node/core/candidate-validation/src/tests.rs index cf36076cf5089..a1353a6014da7 100644 --- a/polkadot/node/core/candidate-validation/src/tests.rs +++ b/polkadot/node/core/candidate-validation/src/tests.rs @@ -377,7 +377,7 @@ impl ValidationBackend for MockValidateCandidateBackend { _pvf: PvfPrepData, _timeout: Duration, _encoded_params: Vec, - _prepare_priority: polkadot_node_core_pvf::Priority, + _prepare_priority: polkadot_node_core_pvf::PreparePriority, _execute_priority: polkadot_node_core_pvf::ExecutePriority, ) -> Result { // This is expected to panic if called more times than expected, indicating an error in the @@ -1064,7 +1064,7 @@ impl ValidationBackend for MockPreCheckBackend { _pvf: PvfPrepData, _timeout: Duration, _encoded_params: Vec, - _prepare_priority: polkadot_node_core_pvf::Priority, + _prepare_priority: polkadot_node_core_pvf::PreparePriority, _execute_priority: polkadot_node_core_pvf::ExecutePriority, ) -> Result { unreachable!() diff --git a/polkadot/node/core/pvf/src/host.rs b/polkadot/node/core/pvf/src/host.rs index 9074b963786c0..c4cc9172d7dde 100644 --- a/polkadot/node/core/pvf/src/host.rs +++ b/polkadot/node/core/pvf/src/host.rs @@ -24,7 +24,7 @@ use crate::{ artifacts::{ArtifactId, ArtifactPathId, ArtifactState, Artifacts, ArtifactsCleanupConfig}, execute::{self, PendingExecutionRequest}, metrics::Metrics, - prepare, ExecutePriority, Priority, SecurityStatus, ValidationError, LOG_TARGET, + prepare, ExecutePriority, PreparePriority, SecurityStatus, ValidationError, LOG_TARGET, }; use always_assert::never; use futures::{ @@ -109,7 +109,7 @@ impl ValidationHost { pvf: PvfPrepData, exec_timeout: Duration, params: Vec, - priority: Priority, + priority: PreparePriority, execute_priority: ExecutePriority, result_tx: ResultSender, ) -> Result<(), String> { @@ -150,7 +150,7 @@ struct ExecutePvfInputs { pvf: PvfPrepData, exec_timeout: Duration, params: Vec, - priority: Priority, + priority: PreparePriority, execute_priority: ExecutePriority, result_tx: ResultSender, } @@ -517,8 +517,11 @@ async fn handle_precheck_pvf( } } else { artifacts.insert_preparing(artifact_id, vec![result_sender]); - send_prepare(prepare_queue, prepare::ToQueue::Enqueue { priority: Priority::Normal, pvf }) - .await?; + send_prepare( + prepare_queue, + prepare::ToQueue::Enqueue { priority: PreparePriority::Normal, pvf }, + ) + .await?; } Ok(()) } @@ -713,7 +716,7 @@ async fn handle_heads_up( send_prepare( prepare_queue, prepare::ToQueue::Enqueue { - priority: Priority::Normal, + priority: PreparePriority::Normal, pvf: active_pvf, }, ) @@ -727,7 +730,7 @@ async fn handle_heads_up( send_prepare( prepare_queue, - prepare::ToQueue::Enqueue { priority: Priority::Normal, pvf: active_pvf }, + prepare::ToQueue::Enqueue { priority: PreparePriority::Normal, pvf: active_pvf }, ) .await?; } @@ -873,7 +876,7 @@ async fn enqueue_prepare_for_execute( prepare_queue: &mut mpsc::Sender, awaiting_prepare: &mut AwaitingPrepare, pvf: PvfPrepData, - priority: Priority, + priority: PreparePriority, artifact_id: ArtifactId, pending_execution_request: PendingExecutionRequest, ) -> Result<(), Fatal> { @@ -1254,7 +1257,7 @@ pub(crate) mod tests { PvfPrepData::from_discriminator(1), TEST_EXECUTION_TIMEOUT, b"pvf1".to_vec(), - Priority::Normal, + PreparePriority::Normal, ExecutePriority::Normal, result_tx, ) @@ -1266,7 +1269,7 @@ pub(crate) mod tests { PvfPrepData::from_discriminator(1), TEST_EXECUTION_TIMEOUT, b"pvf1".to_vec(), - Priority::Critical, + PreparePriority::Critical, ExecutePriority::Normal, result_tx, ) @@ -1278,7 +1281,7 @@ pub(crate) mod tests { PvfPrepData::from_discriminator(2), TEST_EXECUTION_TIMEOUT, b"pvf2".to_vec(), - Priority::Normal, + PreparePriority::Normal, ExecutePriority::Normal, result_tx, ) @@ -1420,7 +1423,7 @@ pub(crate) mod tests { PvfPrepData::from_discriminator(1), TEST_EXECUTION_TIMEOUT, b"pvf2".to_vec(), - Priority::Critical, + PreparePriority::Critical, ExecutePriority::Normal, result_tx, ) @@ -1468,7 +1471,7 @@ pub(crate) mod tests { PvfPrepData::from_discriminator(2), TEST_EXECUTION_TIMEOUT, b"pvf2".to_vec(), - Priority::Critical, + PreparePriority::Critical, ExecutePriority::Normal, result_tx, ) @@ -1571,7 +1574,7 @@ pub(crate) mod tests { PvfPrepData::from_discriminator(1), TEST_EXECUTION_TIMEOUT, b"pvf".to_vec(), - Priority::Critical, + PreparePriority::Critical, ExecutePriority::Normal, result_tx, ) @@ -1602,7 +1605,7 @@ pub(crate) mod tests { PvfPrepData::from_discriminator(1), TEST_EXECUTION_TIMEOUT, b"pvf".to_vec(), - Priority::Critical, + PreparePriority::Critical, ExecutePriority::Normal, result_tx_2, ) @@ -1625,7 +1628,7 @@ pub(crate) mod tests { PvfPrepData::from_discriminator(1), TEST_EXECUTION_TIMEOUT, b"pvf".to_vec(), - Priority::Critical, + PreparePriority::Critical, ExecutePriority::Normal, result_tx_3, ) @@ -1676,7 +1679,7 @@ pub(crate) mod tests { PvfPrepData::from_discriminator(1), TEST_EXECUTION_TIMEOUT, b"pvf".to_vec(), - Priority::Critical, + PreparePriority::Critical, ExecutePriority::Normal, result_tx, ) @@ -1707,7 +1710,7 @@ pub(crate) mod tests { PvfPrepData::from_discriminator(1), TEST_EXECUTION_TIMEOUT, b"pvf".to_vec(), - Priority::Critical, + PreparePriority::Critical, ExecutePriority::Normal, result_tx_2, ) @@ -1730,7 +1733,7 @@ pub(crate) mod tests { PvfPrepData::from_discriminator(1), TEST_EXECUTION_TIMEOUT, b"pvf".to_vec(), - Priority::Critical, + PreparePriority::Critical, ExecutePriority::Normal, result_tx_3, ) @@ -1797,7 +1800,7 @@ pub(crate) mod tests { PvfPrepData::from_discriminator(1), TEST_EXECUTION_TIMEOUT, b"pvf1".to_vec(), - Priority::Normal, + PreparePriority::Normal, ExecutePriority::Normal, result_tx, ) diff --git a/polkadot/node/core/pvf/src/lib.rs b/polkadot/node/core/pvf/src/lib.rs index f5b3aa7b627a4..64a1e1d22fc3a 100644 --- a/polkadot/node/core/pvf/src/lib.rs +++ b/polkadot/node/core/pvf/src/lib.rs @@ -110,7 +110,7 @@ pub use host::{ PREPARE_BINARY_NAME, }; pub use metrics::Metrics; -pub use priority::{ExecutePriority, Priority}; +pub use priority::{ExecutePriority, PreparePriority}; pub use worker_interface::{framed_recv, framed_send, JOB_TIMEOUT_WALL_CLOCK_FACTOR}; // Re-export some common types. diff --git a/polkadot/node/core/pvf/src/prepare/queue.rs b/polkadot/node/core/pvf/src/prepare/queue.rs index c7bfa2f3b21ba..dd9b66095178a 100644 --- a/polkadot/node/core/pvf/src/prepare/queue.rs +++ b/polkadot/node/core/pvf/src/prepare/queue.rs @@ -17,7 +17,7 @@ //! A queue that handles requests for PVF preparation. use super::pool::{self, Worker}; -use crate::{artifacts::ArtifactId, metrics::Metrics, Priority, LOG_TARGET}; +use crate::{artifacts::ArtifactId, metrics::Metrics, PreparePriority, LOG_TARGET}; use always_assert::{always, never}; use futures::{channel::mpsc, stream::StreamExt as _, Future, SinkExt}; use polkadot_node_core_pvf_common::{error::PrepareResult, pvf::PvfPrepData}; @@ -36,7 +36,7 @@ pub enum ToQueue { /// /// Note that it is incorrect to enqueue the same PVF again without first receiving the /// [`FromQueue`] response. - Enqueue { priority: Priority, pvf: PvfPrepData }, + Enqueue { priority: PreparePriority, pvf: PvfPrepData }, } /// A response from queue. @@ -80,7 +80,7 @@ slotmap::new_key_type! { pub struct Job; } struct JobData { /// The priority of this job. Can be bumped. - priority: Priority, + priority: PreparePriority, pvf: PvfPrepData, worker: Option, } @@ -106,18 +106,18 @@ struct Unscheduled { } impl Unscheduled { - fn queue_mut(&mut self, prio: Priority) -> &mut VecDeque { + fn queue_mut(&mut self, prio: PreparePriority) -> &mut VecDeque { match prio { - Priority::Normal => &mut self.normal, - Priority::Critical => &mut self.critical, + PreparePriority::Normal => &mut self.normal, + PreparePriority::Critical => &mut self.critical, } } - fn add(&mut self, prio: Priority, job: Job) { + fn add(&mut self, prio: PreparePriority, job: Job) { self.queue_mut(prio).push_back(job); } - fn readd(&mut self, prio: Priority, job: Job) { + fn readd(&mut self, prio: PreparePriority, job: Job) { self.queue_mut(prio).push_front(job); } @@ -126,8 +126,8 @@ impl Unscheduled { } fn next(&mut self) -> Option { - let mut check = |prio: Priority| self.queue_mut(prio).pop_front(); - check(Priority::Critical).or_else(|| check(Priority::Normal)) + let mut check = |prio: PreparePriority| self.queue_mut(prio).pop_front(); + check(PreparePriority::Critical).or_else(|| check(PreparePriority::Normal)) } } @@ -219,7 +219,7 @@ async fn handle_to_queue(queue: &mut Queue, to_queue: ToQueue) -> Result<(), Fat async fn handle_enqueue( queue: &mut Queue, - priority: Priority, + priority: PreparePriority, pvf: PvfPrepData, ) -> Result<(), Fatal> { gum::debug!( @@ -394,7 +394,7 @@ async fn handle_worker_rip(queue: &mut Queue, worker: Worker) -> Result<(), Fata // this path cannot be hit; // qed. never!("the job of the ripped worker must be known but it is not"); - Priority::Normal + PreparePriority::Normal }); queue.unscheduled.readd(priority, job); } @@ -604,7 +604,7 @@ mod tests { async fn properly_concludes() { let mut test = Test::new(2, 2); - test.send_queue(ToQueue::Enqueue { priority: Priority::Normal, pvf: pvf(1) }); + test.send_queue(ToQueue::Enqueue { priority: PreparePriority::Normal, pvf: pvf(1) }); assert_eq!(test.poll_and_recv_to_pool().await, pool::ToPool::Spawn); let w = test.workers.insert(()); @@ -625,7 +625,7 @@ mod tests { async fn dont_spawn_over_soft_limit_unless_critical() { let mut test = Test::new(2, 3); - let priority = Priority::Normal; + let priority = PreparePriority::Normal; test.send_queue(ToQueue::Enqueue { priority, pvf: PvfPrepData::from_discriminator(1) }); test.send_queue(ToQueue::Enqueue { priority, pvf: PvfPrepData::from_discriminator(2) }); // Start a non-precheck preparation for this one. @@ -658,7 +658,7 @@ mod tests { // Enqueue a critical job. test.send_queue(ToQueue::Enqueue { - priority: Priority::Critical, + priority: PreparePriority::Critical, pvf: PvfPrepData::from_discriminator(4), }); @@ -672,7 +672,7 @@ mod tests { let mut test = Test::new(1, 2); test.send_queue(ToQueue::Enqueue { - priority: Priority::Normal, + priority: PreparePriority::Normal, pvf: PvfPrepData::from_discriminator(1), }); assert_eq!(test.poll_and_recv_to_pool().await, pool::ToPool::Spawn); @@ -682,7 +682,7 @@ mod tests { // Enqueue a critical job, which warrants spawning over the soft limit. test.send_queue(ToQueue::Enqueue { - priority: Priority::Critical, + priority: PreparePriority::Critical, pvf: PvfPrepData::from_discriminator(2), }); assert_eq!(test.poll_and_recv_to_pool().await, pool::ToPool::Spawn); @@ -706,7 +706,7 @@ mod tests { async fn worker_mass_die_out_doesnt_stall_queue() { let mut test = Test::new(2, 2); - let priority = Priority::Normal; + let priority = PreparePriority::Normal; test.send_queue(ToQueue::Enqueue { priority, pvf: PvfPrepData::from_discriminator(1) }); test.send_queue(ToQueue::Enqueue { priority, pvf: PvfPrepData::from_discriminator(2) }); // Start a non-precheck preparation for this one. @@ -748,7 +748,7 @@ mod tests { let mut test = Test::new(2, 2); test.send_queue(ToQueue::Enqueue { - priority: Priority::Normal, + priority: PreparePriority::Normal, pvf: PvfPrepData::from_discriminator(1), }); @@ -772,7 +772,7 @@ mod tests { let mut test = Test::new(2, 2); test.send_queue(ToQueue::Enqueue { - priority: Priority::Normal, + priority: PreparePriority::Normal, pvf: PvfPrepData::from_discriminator(1), }); diff --git a/polkadot/node/core/pvf/src/priority.rs b/polkadot/node/core/pvf/src/priority.rs index d504b036580bf..2c6f363c0cefb 100644 --- a/polkadot/node/core/pvf/src/priority.rs +++ b/polkadot/node/core/pvf/src/priority.rs @@ -16,7 +16,7 @@ /// A priority assigned to preparation of a PVF. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] -pub enum Priority { +pub enum PreparePriority { /// Normal priority for things that do not require immediate response, but still need to be /// done pretty quick. /// @@ -29,10 +29,10 @@ pub enum Priority { Critical, } -impl Priority { +impl PreparePriority { /// Returns `true` if `self` is `Critical` pub fn is_critical(self) -> bool { - self == Priority::Critical + self == PreparePriority::Critical } } diff --git a/polkadot/node/core/pvf/tests/it/main.rs b/polkadot/node/core/pvf/tests/it/main.rs index 0b5064ee75c32..b14e583c768c9 100644 --- a/polkadot/node/core/pvf/tests/it/main.rs +++ b/polkadot/node/core/pvf/tests/it/main.rs @@ -123,7 +123,7 @@ impl TestHost { ), TEST_EXECUTION_TIMEOUT, params.encode(), - polkadot_node_core_pvf::Priority::Normal, + polkadot_node_core_pvf::PreparePriority::Normal, polkadot_node_core_pvf::ExecutePriority::Normal, result_tx, ) From fc7f27a9d83ba4749702dc773484806e7084253e Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 20 Jun 2024 12:25:47 +0200 Subject: [PATCH 06/48] Set low priority for backing --- polkadot/node/core/candidate-validation/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/core/candidate-validation/src/lib.rs b/polkadot/node/core/candidate-validation/src/lib.rs index 242ab5f1b51bf..43e0252b2687c 100644 --- a/polkadot/node/core/candidate-validation/src/lib.rs +++ b/polkadot/node/core/candidate-validation/src/lib.rs @@ -676,7 +676,7 @@ async fn validate_candidate_exhaustive( exec_timeout, params.encode(), polkadot_node_core_pvf::PreparePriority::Normal, - polkadot_node_core_pvf::ExecutePriority::Normal, + polkadot_node_core_pvf::ExecutePriority::Low, ) .await }, From 9b8a5e9289d204655bc77dc6fb41effbecd65196 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Tue, 25 Jun 2024 11:06:19 +0200 Subject: [PATCH 07/48] Add additional option for disputes --- polkadot/node/core/approval-voting/src/lib.rs | 8 ++--- .../node/core/approval-voting/src/tests.rs | 10 +++---- polkadot/node/core/backing/src/lib.rs | 7 +++-- polkadot/node/core/backing/src/tests/mod.rs | 20 ++++++------- .../src/tests/prospective_parachains.rs | 2 +- .../node/core/candidate-validation/src/lib.rs | 24 +++++++-------- .../core/candidate-validation/src/tests.rs | 30 +++++++++---------- .../src/participation/mod.rs | 4 +-- .../src/participation/tests.rs | 10 +++---- polkadot/node/core/pvf/src/priority.rs | 22 ++++++++++++++ polkadot/node/malus/src/variants/common.rs | 2 +- .../node/overseer/examples/minimal-example.rs | 4 +-- polkadot/node/overseer/src/tests.rs | 6 ++-- polkadot/node/subsystem-types/src/messages.rs | 26 ++++++++++++++-- 14 files changed, 110 insertions(+), 65 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index d4b6855a44d0e..c9170ffc46911 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -41,7 +41,7 @@ use polkadot_node_subsystem::{ ApprovalVotingMessage, AssignmentCheckError, AssignmentCheckResult, AvailabilityRecoveryMessage, BlockDescription, CandidateValidationMessage, ChainApiMessage, ChainSelectionMessage, DisputeCoordinatorMessage, HighestApprovedAncestorBlock, - RuntimeApiMessage, RuntimeApiRequest, + PvfExecution, RuntimeApiMessage, RuntimeApiRequest, }, overseer, FromOrchestra, OverseerSignal, SpawnedSubsystem, SubsystemError, SubsystemResult, SubsystemSender, @@ -56,8 +56,8 @@ use polkadot_node_subsystem_util::{ use polkadot_primitives::{ ApprovalVoteMultipleCandidates, ApprovalVotingParams, BlockNumber, CandidateHash, CandidateIndex, CandidateReceipt, CoreIndex, DisputeStatement, ExecutorParams, GroupIndex, - Hash, PvfExecKind, SessionIndex, SessionInfo, ValidDisputeStatementKind, ValidatorId, - ValidatorIndex, ValidatorPair, ValidatorSignature, + Hash, SessionIndex, SessionInfo, ValidDisputeStatementKind, ValidatorId, ValidatorIndex, + ValidatorPair, ValidatorSignature, }; use sc_keystore::LocalKeystore; use sp_application_crypto::Pair; @@ -3529,7 +3529,7 @@ async fn launch_approval( candidate_receipt: candidate.clone(), pov: available_data.pov, executor_params, - exec_kind: PvfExecKind::Approval, + exec_kind: PvfExecution::Approval, response_sender: val_tx, }) .await; diff --git a/polkadot/node/core/approval-voting/src/tests.rs b/polkadot/node/core/approval-voting/src/tests.rs index 64ae86bc013a5..3db5d4591682a 100644 --- a/polkadot/node/core/approval-voting/src/tests.rs +++ b/polkadot/node/core/approval-voting/src/tests.rs @@ -3270,7 +3270,7 @@ async fn handle_double_assignment_import( exec_kind, response_sender, .. - }) if exec_kind == PvfExecKind::Approval => { + }) if exec_kind == PvfExecution::Approval => { response_sender.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) .unwrap(); } @@ -4210,7 +4210,7 @@ async fn handle_approval_on_max_coalesce_count( for _ in &candidate_indices { assert_matches!( overseer_recv(virtual_overseer).await, - AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive{exec_kind, response_sender, ..}) if exec_kind == PvfExecKind::Approval => { + AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive{exec_kind, response_sender, ..}) if exec_kind == PvfExecution::Approval => { response_sender.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) .unwrap(); } @@ -4274,7 +4274,7 @@ async fn handle_approval_on_max_wait_time( for _ in &candidate_indices { assert_matches!( overseer_recv(virtual_overseer).await, - AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive{exec_kind, response_sender, ..}) if exec_kind == PvfExecKind::Approval => { + AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive{exec_kind, response_sender, ..}) if exec_kind == PvfExecution::Approval => { response_sender.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) .unwrap(); } @@ -4777,7 +4777,7 @@ fn subsystem_relaunches_approval_work_on_restart() { exec_kind, response_sender, .. - }) if exec_kind == PvfExecKind::Approval => { + }) if exec_kind == PvfExecution::Approval => { response_sender.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) .unwrap(); } @@ -4897,7 +4897,7 @@ fn subsystem_sends_pending_approvals_on_approval_restart() { exec_kind, response_sender, .. - }) if exec_kind == PvfExecKind::Approval => { + }) if exec_kind == PvfExecution::Approval => { response_sender.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) .unwrap(); } diff --git a/polkadot/node/core/backing/src/lib.rs b/polkadot/node/core/backing/src/lib.rs index 38e8a93bb0482..8be2f2e7e860a 100644 --- a/polkadot/node/core/backing/src/lib.rs +++ b/polkadot/node/core/backing/src/lib.rs @@ -89,8 +89,9 @@ use polkadot_node_subsystem::{ AvailabilityDistributionMessage, AvailabilityStoreMessage, CanSecondRequest, CandidateBackingMessage, CandidateValidationMessage, CollatorProtocolMessage, HypotheticalCandidate, HypotheticalMembershipRequest, IntroduceSecondedCandidateRequest, - ProspectiveParachainsMessage, ProvisionableData, ProvisionerMessage, RuntimeApiMessage, - RuntimeApiRequest, StatementDistributionMessage, StoreAvailableDataError, + ProspectiveParachainsMessage, ProvisionableData, ProvisionerMessage, PvfExecution, + RuntimeApiMessage, RuntimeApiRequest, StatementDistributionMessage, + StoreAvailableDataError, }, overseer, ActiveLeavesUpdate, FromOrchestra, OverseerSignal, SpawnedSubsystem, SubsystemError, }; @@ -631,7 +632,7 @@ async fn request_candidate_validation( candidate_receipt, pov, executor_params, - exec_kind: PvfExecKind::Backing, + exec_kind: PvfExecution::Backing, response_sender: tx, }) .await; diff --git a/polkadot/node/core/backing/src/tests/mod.rs b/polkadot/node/core/backing/src/tests/mod.rs index bb23c7fbeb24f..2f393a5685592 100644 --- a/polkadot/node/core/backing/src/tests/mod.rs +++ b/polkadot/node/core/backing/src/tests/mod.rs @@ -405,7 +405,7 @@ async fn assert_validate_from_exhaustive( ) if validation_data == *assert_pvd && validation_code == *assert_validation_code && *pov == *assert_pov && &candidate_receipt.descriptor == assert_candidate.descriptor() && - exec_kind == PvfExecKind::Backing && + exec_kind == PvfExecution::Backing && candidate_receipt.commitments_hash == assert_candidate.commitments.hash() => { response_sender.send(Ok(ValidationResult::Valid( @@ -622,7 +622,7 @@ fn backing_works(#[case] elastic_scaling_mvp: bool) { ) if validation_data == pvd_ab && validation_code == validation_code_ab && *pov == pov_ab && &candidate_receipt.descriptor == candidate_a.descriptor() && - exec_kind == PvfExecKind::Backing && + exec_kind == PvfExecution::Backing && candidate_receipt.commitments_hash == candidate_a_commitments_hash => { response_sender.send(Ok( @@ -1250,7 +1250,7 @@ fn backing_works_while_validation_ongoing() { ) if validation_data == pvd_abc && validation_code == validation_code_abc && *pov == pov_abc && &candidate_receipt.descriptor == candidate_a.descriptor() && - exec_kind == PvfExecKind::Backing && + exec_kind == PvfExecution::Backing && candidate_a_commitments_hash == candidate_receipt.commitments_hash => { // we never validate the candidate. our local node @@ -1417,7 +1417,7 @@ fn backing_misbehavior_works() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_a && &candidate_receipt.descriptor == candidate_a.descriptor() && - exec_kind == PvfExecKind::Backing && + exec_kind == PvfExecution::Backing && candidate_a_commitments_hash == candidate_receipt.commitments_hash => { response_sender.send(Ok( @@ -1584,7 +1584,7 @@ fn backing_dont_second_invalid() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_block_a && &candidate_receipt.descriptor == candidate_a.descriptor() && - exec_kind == PvfExecKind::Backing && + exec_kind == PvfExecution::Backing && candidate_a.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Ok(ValidationResult::Invalid(InvalidCandidate::BadReturn))).unwrap(); @@ -1624,7 +1624,7 @@ fn backing_dont_second_invalid() { ) if validation_data == pvd_b && validation_code == validation_code_b && *pov == pov_block_b && &candidate_receipt.descriptor == candidate_b.descriptor() && - exec_kind == PvfExecKind::Backing && + exec_kind == PvfExecution::Backing && candidate_b.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Ok( @@ -1751,7 +1751,7 @@ fn backing_second_after_first_fails_works() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_a && &candidate_receipt.descriptor == candidate.descriptor() && - exec_kind == PvfExecKind::Backing && + exec_kind == PvfExecution::Backing && candidate.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Ok(ValidationResult::Invalid(InvalidCandidate::BadReturn))).unwrap(); @@ -1895,7 +1895,7 @@ fn backing_works_after_failed_validation() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_a && &candidate_receipt.descriptor == candidate.descriptor() && - exec_kind == PvfExecKind::Backing && + exec_kind == PvfExecution::Backing && candidate.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Err(ValidationFailed("Internal test error".into()))).unwrap(); @@ -2174,7 +2174,7 @@ fn retry_works() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_a && &candidate_receipt.descriptor == candidate.descriptor() && - exec_kind == PvfExecKind::Backing && + exec_kind == PvfExecution::Backing && candidate.commitments.hash() == candidate_receipt.commitments_hash ); virtual_overseer @@ -2716,7 +2716,7 @@ fn validator_ignores_statements_from_disabled_validators() { ) if validation_data == pvd && validation_code == expected_validation_code && *pov == expected_pov && &candidate_receipt.descriptor == candidate.descriptor() && - exec_kind == PvfExecKind::Backing && + exec_kind == PvfExecution::Backing && candidate_commitments_hash == candidate_receipt.commitments_hash => { response_sender.send(Ok( diff --git a/polkadot/node/core/backing/src/tests/prospective_parachains.rs b/polkadot/node/core/backing/src/tests/prospective_parachains.rs index 74490c84eb18b..f491f22300514 100644 --- a/polkadot/node/core/backing/src/tests/prospective_parachains.rs +++ b/polkadot/node/core/backing/src/tests/prospective_parachains.rs @@ -256,7 +256,7 @@ async fn assert_validate_seconded_candidate( &validation_code == assert_validation_code && &*pov == assert_pov && &candidate_receipt.descriptor == candidate.descriptor() && - exec_kind == PvfExecKind::Backing && + exec_kind == PvfExecution::Backing && candidate.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Ok(ValidationResult::Valid( diff --git a/polkadot/node/core/candidate-validation/src/lib.rs b/polkadot/node/core/candidate-validation/src/lib.rs index 43e0252b2687c..be447c189e0b5 100644 --- a/polkadot/node/core/candidate-validation/src/lib.rs +++ b/polkadot/node/core/candidate-validation/src/lib.rs @@ -33,8 +33,8 @@ use polkadot_node_primitives::{ use polkadot_node_subsystem::{ errors::RuntimeApiError, messages::{ - CandidateValidationMessage, PreCheckOutcome, RuntimeApiMessage, RuntimeApiRequest, - ValidationFailed, + CandidateValidationMessage, PreCheckOutcome, PvfExecution, RuntimeApiMessage, + RuntimeApiRequest, ValidationFailed, }, overseer, FromOrchestra, OverseerSignal, SpawnedSubsystem, SubsystemError, SubsystemResult, SubsystemSender, @@ -542,7 +542,7 @@ async fn validate_from_chain_state( candidate_receipt: CandidateReceipt, pov: Arc, executor_params: ExecutorParams, - exec_kind: PvfExecKind, + exec_kind: PvfExecution, metrics: &Metrics, ) -> Result where @@ -598,7 +598,7 @@ async fn validate_candidate_exhaustive( candidate_receipt: CandidateReceipt, pov: Arc, executor_params: ExecutorParams, - exec_kind: PvfExecKind, + exec_kind: PvfExecution, metrics: &Metrics, ) -> Result { let _timer = metrics.time_validate_candidate_exhaustive(); @@ -660,9 +660,9 @@ async fn validate_candidate_exhaustive( let result = match exec_kind { // Retry is disabled to reduce the chance of nondeterministic blocks getting backed and // honest backers getting slashed. - PvfExecKind::Backing => { + PvfExecution::Backing => { let prep_timeout = pvf_prep_timeout(&executor_params, PvfPrepKind::Prepare); - let exec_timeout = pvf_exec_timeout(&executor_params, exec_kind); + let exec_timeout = pvf_exec_timeout(&executor_params, exec_kind.into()); let pvf = PvfPrepData::from_code( raw_validation_code.to_vec(), executor_params, @@ -675,21 +675,21 @@ async fn validate_candidate_exhaustive( pvf, exec_timeout, params.encode(), - polkadot_node_core_pvf::PreparePriority::Normal, - polkadot_node_core_pvf::ExecutePriority::Low, + exec_kind.into(), + exec_kind.into(), ) .await }, - PvfExecKind::Approval => + PvfExecution::Approval | PvfExecution::Dispute => validation_backend .validate_candidate_with_retry( raw_validation_code.to_vec(), - pvf_exec_timeout(&executor_params, exec_kind), + pvf_exec_timeout(&executor_params, exec_kind.into()), params, executor_params, PVF_APPROVAL_EXECUTION_RETRY_DELAY, - polkadot_node_core_pvf::PreparePriority::Critical, - polkadot_node_core_pvf::ExecutePriority::Normal, + exec_kind.into(), + exec_kind.into(), ) .await, }; diff --git a/polkadot/node/core/candidate-validation/src/tests.rs b/polkadot/node/core/candidate-validation/src/tests.rs index a1353a6014da7..b15db1479bad5 100644 --- a/polkadot/node/core/candidate-validation/src/tests.rs +++ b/polkadot/node/core/candidate-validation/src/tests.rs @@ -447,7 +447,7 @@ fn candidate_validation_ok_is_ok() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecKind::Backing, + PvfExecution::Backing, &Default::default(), )) .unwrap(); @@ -499,7 +499,7 @@ fn candidate_validation_bad_return_is_invalid() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecKind::Backing, + PvfExecution::Backing, &Default::default(), )) .unwrap(); @@ -580,7 +580,7 @@ fn candidate_validation_one_ambiguous_error_is_valid() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecKind::Approval, + PvfExecution::Approval, &Default::default(), )) .unwrap(); @@ -621,7 +621,7 @@ fn candidate_validation_multiple_ambiguous_errors_is_invalid() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecKind::Approval, + PvfExecution::Approval, &Default::default(), )) .unwrap(); @@ -633,7 +633,7 @@ fn candidate_validation_multiple_ambiguous_errors_is_invalid() { #[test] fn candidate_validation_retry_internal_errors() { let v = candidate_validation_retry_on_error_helper( - PvfExecKind::Approval, + PvfExecution::Approval, vec![ Err(InternalValidationError::HostCommunication("foo".into()).into()), // Throw an AJD error, we should still retry again. @@ -651,7 +651,7 @@ fn candidate_validation_retry_internal_errors() { #[test] fn candidate_validation_dont_retry_internal_errors() { let v = candidate_validation_retry_on_error_helper( - PvfExecKind::Backing, + PvfExecution::Backing, vec![ Err(InternalValidationError::HostCommunication("foo".into()).into()), // Throw an AWD error, we should still retry again. @@ -668,7 +668,7 @@ fn candidate_validation_dont_retry_internal_errors() { #[test] fn candidate_validation_retry_panic_errors() { let v = candidate_validation_retry_on_error_helper( - PvfExecKind::Approval, + PvfExecution::Approval, vec![ Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::JobError("foo".into()))), // Throw an AWD error, we should still retry again. @@ -685,7 +685,7 @@ fn candidate_validation_retry_panic_errors() { #[test] fn candidate_validation_dont_retry_panic_errors() { let v = candidate_validation_retry_on_error_helper( - PvfExecKind::Backing, + PvfExecution::Backing, vec![ Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::JobError("foo".into()))), // Throw an AWD error, we should still retry again. @@ -699,7 +699,7 @@ fn candidate_validation_dont_retry_panic_errors() { } fn candidate_validation_retry_on_error_helper( - exec_kind: PvfExecKind, + exec_kind: PvfExecution, mock_errors: Vec>, ) -> Result { let validation_data = PersistedValidationData { max_pov_size: 1024, ..Default::default() }; @@ -777,7 +777,7 @@ fn candidate_validation_timeout_is_internal_error() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecKind::Backing, + PvfExecution::Backing, &Default::default(), )); @@ -822,7 +822,7 @@ fn candidate_validation_commitment_hash_mismatch_is_invalid() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecKind::Backing, + PvfExecution::Backing, &Default::default(), )) .unwrap(); @@ -874,7 +874,7 @@ fn candidate_validation_code_mismatch_is_invalid() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecKind::Backing, + PvfExecution::Backing, &Default::default(), )) .unwrap(); @@ -931,7 +931,7 @@ fn compressed_code_works() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecKind::Backing, + PvfExecution::Backing, &Default::default(), )); @@ -985,7 +985,7 @@ fn code_decompression_failure_is_error() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecKind::Backing, + PvfExecution::Backing, &Default::default(), )); @@ -1040,7 +1040,7 @@ fn pov_decompression_failure_is_invalid() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecKind::Backing, + PvfExecution::Backing, &Default::default(), )); diff --git a/polkadot/node/core/dispute-coordinator/src/participation/mod.rs b/polkadot/node/core/dispute-coordinator/src/participation/mod.rs index b58ce570f8fff..372d111c86e46 100644 --- a/polkadot/node/core/dispute-coordinator/src/participation/mod.rs +++ b/polkadot/node/core/dispute-coordinator/src/participation/mod.rs @@ -27,7 +27,7 @@ use futures_timer::Delay; use polkadot_node_primitives::ValidationResult; use polkadot_node_subsystem::{ - messages::{AvailabilityRecoveryMessage, CandidateValidationMessage}, + messages::{AvailabilityRecoveryMessage, CandidateValidationMessage, PvfExecution}, overseer, ActiveLeavesUpdate, RecoveryError, }; use polkadot_node_subsystem_util::runtime::get_validation_code_by_hash; @@ -387,7 +387,7 @@ async fn participate( candidate_receipt: req.candidate_receipt().clone(), pov: available_data.pov, executor_params: req.executor_params(), - exec_kind: PvfExecKind::Approval, + exec_kind: PvfExecution::Approval, response_sender: validation_tx, }) .await; diff --git a/polkadot/node/core/dispute-coordinator/src/participation/tests.rs b/polkadot/node/core/dispute-coordinator/src/participation/tests.rs index a80553828ac69..3999e4a9688ad 100644 --- a/polkadot/node/core/dispute-coordinator/src/participation/tests.rs +++ b/polkadot/node/core/dispute-coordinator/src/participation/tests.rs @@ -26,7 +26,7 @@ use codec::Encode; use polkadot_node_primitives::{AvailableData, BlockData, InvalidCandidate, PoV}; use polkadot_node_subsystem::{ messages::{ - AllMessages, ChainApiMessage, DisputeCoordinatorMessage, RuntimeApiMessage, + AllMessages, ChainApiMessage, DisputeCoordinatorMessage, PvfExecution, RuntimeApiMessage, RuntimeApiRequest, }, ActiveLeavesUpdate, SpawnGlue, @@ -116,7 +116,7 @@ pub async fn participation_full_happy_path( ctx_handle.recv().await, AllMessages::CandidateValidation( CandidateValidationMessage::ValidateFromExhaustive { candidate_receipt, exec_kind, response_sender, .. } - ) if exec_kind == PvfExecKind::Approval => { + ) if exec_kind == PvfExecution::Dispute => { if expected_commitments_hash != candidate_receipt.commitments_hash { response_sender.send(Ok(ValidationResult::Invalid(InvalidCandidate::CommitmentsHashMismatch))).unwrap(); } else { @@ -450,7 +450,7 @@ fn cast_invalid_vote_if_validation_fails_or_is_invalid() { ctx_handle.recv().await, AllMessages::CandidateValidation( CandidateValidationMessage::ValidateFromExhaustive { exec_kind, response_sender, .. } - ) if exec_kind == PvfExecKind::Approval => { + ) if exec_kind == PvfExecution::Dispute => { response_sender.send(Ok(ValidationResult::Invalid(InvalidCandidate::Timeout))).unwrap(); }, "overseer did not receive candidate validation message", @@ -487,7 +487,7 @@ fn cast_invalid_vote_if_commitments_dont_match() { ctx_handle.recv().await, AllMessages::CandidateValidation( CandidateValidationMessage::ValidateFromExhaustive { exec_kind, response_sender, .. } - ) if exec_kind == PvfExecKind::Approval => { + ) if exec_kind == PvfExecution::Dispute => { response_sender.send(Ok(ValidationResult::Invalid(InvalidCandidate::CommitmentsHashMismatch))).unwrap(); }, "overseer did not receive candidate validation message", @@ -524,7 +524,7 @@ fn cast_valid_vote_if_validation_passes() { ctx_handle.recv().await, AllMessages::CandidateValidation( CandidateValidationMessage::ValidateFromExhaustive { exec_kind, response_sender, .. } - ) if exec_kind == PvfExecKind::Approval => { + ) if exec_kind == PvfExecution::Dispute => { response_sender.send(Ok(ValidationResult::Valid(dummy_candidate_commitments(None), PersistedValidationData::default()))).unwrap(); }, "overseer did not receive candidate validation message", diff --git a/polkadot/node/core/pvf/src/priority.rs b/polkadot/node/core/pvf/src/priority.rs index 2c6f363c0cefb..24146f546a254 100644 --- a/polkadot/node/core/pvf/src/priority.rs +++ b/polkadot/node/core/pvf/src/priority.rs @@ -14,6 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +use polkadot_node_subsystem::messages::PvfExecution; + /// A priority assigned to preparation of a PVF. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] pub enum PreparePriority { @@ -36,6 +38,16 @@ impl PreparePriority { } } +impl From for PreparePriority { + fn from(priority: PvfExecution) -> Self { + match priority { + PvfExecution::Backing => PreparePriority::Normal, + PvfExecution::Approval => PreparePriority::Critical, + PvfExecution::Dispute => PreparePriority::Critical, + } + } +} + /// A priority assigned to execution of a PVF. #[derive(Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Clone, Copy)] pub enum ExecutePriority { @@ -47,6 +59,16 @@ pub enum ExecutePriority { Critical, } +impl From for ExecutePriority { + fn from(priority: PvfExecution) -> Self { + match priority { + PvfExecution::Backing => ExecutePriority::Low, + PvfExecution::Approval => ExecutePriority::Normal, + PvfExecution::Dispute => ExecutePriority::Critical, + } + } +} + impl ExecutePriority { /// Returns an iterator over the variants of `ExecutePriority` in order from `Low` to /// `Critical`. diff --git a/polkadot/node/malus/src/variants/common.rs b/polkadot/node/malus/src/variants/common.rs index eb6988f818116..850b99519e9d5 100644 --- a/polkadot/node/malus/src/variants/common.rs +++ b/polkadot/node/malus/src/variants/common.rs @@ -86,7 +86,7 @@ impl FakeCandidateValidation { } } - fn should_misbehave(&self, timeout: PvfExecKind) -> bool { + fn should_misbehave(&self, timeout: PvfExecution) -> bool { match timeout { PvfExecKind::Backing => self.includes_backing(), PvfExecKind::Approval => self.includes_approval(), diff --git a/polkadot/node/overseer/examples/minimal-example.rs b/polkadot/node/overseer/examples/minimal-example.rs index 86a1801a5f2d8..5eca47b243111 100644 --- a/polkadot/node/overseer/examples/minimal-example.rs +++ b/polkadot/node/overseer/examples/minimal-example.rs @@ -24,7 +24,7 @@ use orchestra::async_trait; use std::time::Duration; use polkadot_node_primitives::{BlockData, PoV}; -use polkadot_node_subsystem_types::messages::CandidateValidationMessage; +use polkadot_node_subsystem_types::messages::{CandidateValidationMessage, PvfExecution}; use polkadot_overseer::{ self as overseer, dummy::dummy_overseer_builder, @@ -77,7 +77,7 @@ impl Subsystem1 { candidate_receipt, pov: PoV { block_data: BlockData(Vec::new()) }.into(), executor_params: Default::default(), - exec_kind: PvfExecKind::Backing, + exec_kind: PvfExecution::Backing, response_sender: tx, }; ctx.send_message(msg).await; diff --git a/polkadot/node/overseer/src/tests.rs b/polkadot/node/overseer/src/tests.rs index 177e3addf368d..8325f65559179 100644 --- a/polkadot/node/overseer/src/tests.rs +++ b/polkadot/node/overseer/src/tests.rs @@ -25,7 +25,7 @@ use polkadot_node_primitives::{ }; use polkadot_node_subsystem_test_helpers::mock::{dummy_unpin_handle, new_leaf}; use polkadot_node_subsystem_types::messages::{ - NetworkBridgeEvent, ReportPeerMessage, RuntimeApiRequest, + NetworkBridgeEvent, PvfExecution, ReportPeerMessage, RuntimeApiRequest, }; use polkadot_primitives::{ CandidateHash, CandidateReceipt, CollatorPair, Id as ParaId, InvalidDisputeStatementKind, @@ -108,7 +108,7 @@ where candidate_receipt, pov: PoV { block_data: BlockData(Vec::new()) }.into(), executor_params: Default::default(), - exec_kind: PvfExecKind::Backing, + exec_kind: PvfExecution::Backing, response_sender: tx, }) .await; @@ -806,7 +806,7 @@ fn test_candidate_validation_msg() -> CandidateValidationMessage { candidate_receipt, pov, executor_params: Default::default(), - exec_kind: PvfExecKind::Backing, + exec_kind: PvfExecution::Backing, response_sender, } } diff --git a/polkadot/node/subsystem-types/src/messages.rs b/polkadot/node/subsystem-types/src/messages.rs index 722a97989bce0..f0a9cceab9e00 100644 --- a/polkadot/node/subsystem-types/src/messages.rs +++ b/polkadot/node/subsystem-types/src/messages.rs @@ -160,7 +160,7 @@ pub enum CandidateValidationMessage { /// Session's executor parameters executor_params: ExecutorParams, /// Execution kind, used for timeouts and retries (backing/approvals) - exec_kind: PvfExecKind, + exec_kind: PvfExecution, /// The sending side of the response channel response_sender: oneshot::Sender>, }, @@ -185,7 +185,7 @@ pub enum CandidateValidationMessage { /// Session's executor parameters executor_params: ExecutorParams, /// Execution kind, used for timeouts and retries (backing/approvals) - exec_kind: PvfExecKind, + exec_kind: PvfExecution, /// The sending side of the response channel response_sender: oneshot::Sender>, }, @@ -204,6 +204,28 @@ pub enum CandidateValidationMessage { }, } +/// Extends primitives::PvfExecKind to have a separate value for duspute requests +/// which is important for prioritization. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PvfExecution { + /// For backing requests. + Backing, + /// For approval requests + Approval, + /// For dispute requests + Dispute, +} + +impl From for PvfExecKind { + fn from(exec: PvfExecution) -> Self { + match exec { + PvfExecution::Backing => PvfExecKind::Backing, + PvfExecution::Approval => PvfExecKind::Approval, + PvfExecution::Dispute => PvfExecKind::Approval, + } + } +} + /// Messages received by the Collator Protocol subsystem. #[derive(Debug, derive_more::From)] pub enum CollatorProtocolMessage { From 610f0cc318a37240ff587516f32220647cc53538 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Tue, 25 Jun 2024 11:12:00 +0200 Subject: [PATCH 08/48] Fix clippy --- polkadot/node/core/backing/src/lib.rs | 4 ++-- polkadot/node/core/backing/src/tests/mod.rs | 2 +- .../node/core/dispute-coordinator/src/participation/mod.rs | 4 +--- polkadot/node/overseer/examples/minimal-example.rs | 2 +- polkadot/node/overseer/src/tests.rs | 2 +- 5 files changed, 6 insertions(+), 8 deletions(-) diff --git a/polkadot/node/core/backing/src/lib.rs b/polkadot/node/core/backing/src/lib.rs index 8be2f2e7e860a..261b8bd327119 100644 --- a/polkadot/node/core/backing/src/lib.rs +++ b/polkadot/node/core/backing/src/lib.rs @@ -109,8 +109,8 @@ use polkadot_primitives::{ node_features::FeatureIndex, BackedCandidate, CandidateCommitments, CandidateHash, CandidateReceipt, CommittedCandidateReceipt, CoreIndex, CoreState, ExecutorParams, GroupIndex, GroupRotationInfo, Hash, Id as ParaId, IndexedVec, NodeFeatures, PersistedValidationData, - PvfExecKind, SessionIndex, SigningContext, ValidationCode, ValidatorId, ValidatorIndex, - ValidatorSignature, ValidityAttestation, + SessionIndex, SigningContext, ValidationCode, ValidatorId, ValidatorIndex, ValidatorSignature, + ValidityAttestation, }; use polkadot_statement_table::{ generic::AttestedCandidate as TableAttestedCandidate, diff --git a/polkadot/node/core/backing/src/tests/mod.rs b/polkadot/node/core/backing/src/tests/mod.rs index 2f393a5685592..56d5a744c90c2 100644 --- a/polkadot/node/core/backing/src/tests/mod.rs +++ b/polkadot/node/core/backing/src/tests/mod.rs @@ -30,7 +30,7 @@ use polkadot_node_subsystem::{ use polkadot_node_subsystem_test_helpers as test_helpers; use polkadot_primitives::{ node_features, CandidateDescriptor, GroupRotationInfo, HeadData, PersistedValidationData, - PvfExecKind, ScheduledCore, SessionIndex, LEGACY_MIN_BACKING_VOTES, + ScheduledCore, SessionIndex, LEGACY_MIN_BACKING_VOTES, }; use polkadot_primitives_test_helpers::{ dummy_candidate_receipt_bad_sig, dummy_collator, dummy_collator_signature, diff --git a/polkadot/node/core/dispute-coordinator/src/participation/mod.rs b/polkadot/node/core/dispute-coordinator/src/participation/mod.rs index 372d111c86e46..03b91cebc1402 100644 --- a/polkadot/node/core/dispute-coordinator/src/participation/mod.rs +++ b/polkadot/node/core/dispute-coordinator/src/participation/mod.rs @@ -31,9 +31,7 @@ use polkadot_node_subsystem::{ overseer, ActiveLeavesUpdate, RecoveryError, }; use polkadot_node_subsystem_util::runtime::get_validation_code_by_hash; -use polkadot_primitives::{ - BlockNumber, CandidateHash, CandidateReceipt, Hash, PvfExecKind, SessionIndex, -}; +use polkadot_primitives::{BlockNumber, CandidateHash, CandidateReceipt, Hash, SessionIndex}; use crate::LOG_TARGET; diff --git a/polkadot/node/overseer/examples/minimal-example.rs b/polkadot/node/overseer/examples/minimal-example.rs index 5eca47b243111..e739311779d52 100644 --- a/polkadot/node/overseer/examples/minimal-example.rs +++ b/polkadot/node/overseer/examples/minimal-example.rs @@ -31,7 +31,7 @@ use polkadot_overseer::{ gen::{FromOrchestra, SpawnedSubsystem}, HeadSupportsParachains, SubsystemError, }; -use polkadot_primitives::{CandidateReceipt, Hash, PvfExecKind}; +use polkadot_primitives::{CandidateReceipt, Hash}; use polkadot_primitives_test_helpers::{dummy_candidate_descriptor, dummy_hash}; struct AlwaysSupportsParachains; diff --git a/polkadot/node/overseer/src/tests.rs b/polkadot/node/overseer/src/tests.rs index 8325f65559179..506501a2e7075 100644 --- a/polkadot/node/overseer/src/tests.rs +++ b/polkadot/node/overseer/src/tests.rs @@ -29,7 +29,7 @@ use polkadot_node_subsystem_types::messages::{ }; use polkadot_primitives::{ CandidateHash, CandidateReceipt, CollatorPair, Id as ParaId, InvalidDisputeStatementKind, - PvfExecKind, SessionIndex, ValidDisputeStatementKind, ValidatorIndex, + SessionIndex, ValidDisputeStatementKind, ValidatorIndex, }; use polkadot_primitives_test_helpers::{ dummy_candidate_descriptor, dummy_candidate_receipt, dummy_hash, From 507113c693ef1d1f32fb1e3ca561b90a126c69c5 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Tue, 25 Jun 2024 11:18:54 +0200 Subject: [PATCH 09/48] Fix types --- polkadot/node/malus/src/variants/common.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/polkadot/node/malus/src/variants/common.rs b/polkadot/node/malus/src/variants/common.rs index 850b99519e9d5..8363379adb31b 100644 --- a/polkadot/node/malus/src/variants/common.rs +++ b/polkadot/node/malus/src/variants/common.rs @@ -86,7 +86,7 @@ impl FakeCandidateValidation { } } - fn should_misbehave(&self, timeout: PvfExecution) -> bool { + fn should_misbehave(&self, timeout: PvfExecKind) -> bool { match timeout { PvfExecKind::Backing => self.includes_backing(), PvfExecKind::Approval => self.includes_approval(), @@ -281,7 +281,7 @@ where }, } => { match self.fake_validation { - x if x.misbehaves_valid() && x.should_misbehave(exec_kind) => { + x if x.misbehaves_valid() && x.should_misbehave(exec_kind.into()) => { // Behave normally if the `PoV` is not known to be malicious. if pov.block_data.0.as_slice() != MALICIOUS_POV { return Some(FromOrchestra::Communication { @@ -336,7 +336,7 @@ where }, } }, - x if x.misbehaves_invalid() && x.should_misbehave(exec_kind) => { + x if x.misbehaves_invalid() && x.should_misbehave(exec_kind.into()) => { // Set the validation result to invalid with probability `p` and trigger a // dispute let behave_maliciously = self.distribution.sample(&mut rand::thread_rng()); @@ -403,7 +403,7 @@ where }, } => { match self.fake_validation { - x if x.misbehaves_valid() && x.should_misbehave(exec_kind) => { + x if x.misbehaves_valid() && x.should_misbehave(exec_kind.into()) => { // Behave normally if the `PoV` is not known to be malicious. if pov.block_data.0.as_slice() != MALICIOUS_POV { return Some(FromOrchestra::Communication { @@ -447,7 +447,7 @@ where }), } }, - x if x.misbehaves_invalid() && x.should_misbehave(exec_kind) => { + x if x.misbehaves_invalid() && x.should_misbehave(exec_kind.into()) => { // Maliciously set the validation result to invalid for a valid candidate // with probability `p` let behave_maliciously = self.distribution.sample(&mut rand::thread_rng()); From 3fbecab4b67ea8e32e1612025579e65180458e41 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Tue, 25 Jun 2024 14:23:27 +0200 Subject: [PATCH 10/48] Fix type --- polkadot/node/core/dispute-coordinator/src/participation/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/core/dispute-coordinator/src/participation/mod.rs b/polkadot/node/core/dispute-coordinator/src/participation/mod.rs index 03b91cebc1402..fbf5c24ccd162 100644 --- a/polkadot/node/core/dispute-coordinator/src/participation/mod.rs +++ b/polkadot/node/core/dispute-coordinator/src/participation/mod.rs @@ -385,7 +385,7 @@ async fn participate( candidate_receipt: req.candidate_receipt().clone(), pov: available_data.pov, executor_params: req.executor_params(), - exec_kind: PvfExecution::Approval, + exec_kind: PvfExecution::Dispute, response_sender: validation_tx, }) .await; From 919260287051b01d7d2795b9132c12f11c59debe Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Wed, 26 Jun 2024 11:55:52 +0200 Subject: [PATCH 11/48] Update polkadot/node/core/pvf/src/priority.rs Co-authored-by: Andrei Sandu <54316454+sandreim@users.noreply.github.com> --- polkadot/node/core/pvf/src/priority.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/core/pvf/src/priority.rs b/polkadot/node/core/pvf/src/priority.rs index 24146f546a254..6a7e13248496d 100644 --- a/polkadot/node/core/pvf/src/priority.rs +++ b/polkadot/node/core/pvf/src/priority.rs @@ -53,7 +53,7 @@ impl From for PreparePriority { pub enum ExecutePriority { /// Low Low, - /// Normall + /// Normal Normal, /// Critical Critical, From bafb6c2b3e0b6726c917e29310dfb68c28614bed Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Wed, 26 Jun 2024 12:40:20 +0200 Subject: [PATCH 12/48] Update --- polkadot/node/core/approval-voting/src/lib.rs | 4 +-- .../node/core/approval-voting/src/tests.rs | 10 +++--- polkadot/node/core/backing/src/lib.rs | 4 +-- polkadot/node/core/backing/src/tests/mod.rs | 20 ++++++------ .../src/tests/prospective_parachains.rs | 2 +- .../node/core/candidate-validation/src/lib.rs | 10 +++--- .../core/candidate-validation/src/tests.rs | 30 ++++++++--------- .../src/participation/mod.rs | 4 +-- .../src/participation/tests.rs | 12 +++---- polkadot/node/core/pvf/src/execute/queue.rs | 32 +++++++++++-------- polkadot/node/core/pvf/src/priority.rs | 22 ++++++------- .../node/overseer/examples/minimal-example.rs | 4 +-- polkadot/node/overseer/src/tests.rs | 6 ++-- polkadot/node/subsystem-types/src/messages.rs | 16 +++++----- 14 files changed, 91 insertions(+), 85 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index c9170ffc46911..de966f3ddb0c8 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -41,7 +41,7 @@ use polkadot_node_subsystem::{ ApprovalVotingMessage, AssignmentCheckError, AssignmentCheckResult, AvailabilityRecoveryMessage, BlockDescription, CandidateValidationMessage, ChainApiMessage, ChainSelectionMessage, DisputeCoordinatorMessage, HighestApprovedAncestorBlock, - PvfExecution, RuntimeApiMessage, RuntimeApiRequest, + PvfExecutionPriority, RuntimeApiMessage, RuntimeApiRequest, }, overseer, FromOrchestra, OverseerSignal, SpawnedSubsystem, SubsystemError, SubsystemResult, SubsystemSender, @@ -3529,7 +3529,7 @@ async fn launch_approval( candidate_receipt: candidate.clone(), pov: available_data.pov, executor_params, - exec_kind: PvfExecution::Approval, + exec_kind: PvfExecutionPriority::Approval, response_sender: val_tx, }) .await; diff --git a/polkadot/node/core/approval-voting/src/tests.rs b/polkadot/node/core/approval-voting/src/tests.rs index 3db5d4591682a..766ed6fb5cc5f 100644 --- a/polkadot/node/core/approval-voting/src/tests.rs +++ b/polkadot/node/core/approval-voting/src/tests.rs @@ -3270,7 +3270,7 @@ async fn handle_double_assignment_import( exec_kind, response_sender, .. - }) if exec_kind == PvfExecution::Approval => { + }) if exec_kind == PvfExecutionPriority::Approval => { response_sender.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) .unwrap(); } @@ -4210,7 +4210,7 @@ async fn handle_approval_on_max_coalesce_count( for _ in &candidate_indices { assert_matches!( overseer_recv(virtual_overseer).await, - AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive{exec_kind, response_sender, ..}) if exec_kind == PvfExecution::Approval => { + AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive{exec_kind, response_sender, ..}) if exec_kind == PvfExecutionPriority::Approval => { response_sender.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) .unwrap(); } @@ -4274,7 +4274,7 @@ async fn handle_approval_on_max_wait_time( for _ in &candidate_indices { assert_matches!( overseer_recv(virtual_overseer).await, - AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive{exec_kind, response_sender, ..}) if exec_kind == PvfExecution::Approval => { + AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive{exec_kind, response_sender, ..}) if exec_kind == PvfExecutionPriority::Approval => { response_sender.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) .unwrap(); } @@ -4777,7 +4777,7 @@ fn subsystem_relaunches_approval_work_on_restart() { exec_kind, response_sender, .. - }) if exec_kind == PvfExecution::Approval => { + }) if exec_kind == PvfExecutionPriority::Approval => { response_sender.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) .unwrap(); } @@ -4897,7 +4897,7 @@ fn subsystem_sends_pending_approvals_on_approval_restart() { exec_kind, response_sender, .. - }) if exec_kind == PvfExecution::Approval => { + }) if exec_kind == PvfExecutionPriority::Approval => { response_sender.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) .unwrap(); } diff --git a/polkadot/node/core/backing/src/lib.rs b/polkadot/node/core/backing/src/lib.rs index 261b8bd327119..fd8d6d8fedbbf 100644 --- a/polkadot/node/core/backing/src/lib.rs +++ b/polkadot/node/core/backing/src/lib.rs @@ -89,7 +89,7 @@ use polkadot_node_subsystem::{ AvailabilityDistributionMessage, AvailabilityStoreMessage, CanSecondRequest, CandidateBackingMessage, CandidateValidationMessage, CollatorProtocolMessage, HypotheticalCandidate, HypotheticalMembershipRequest, IntroduceSecondedCandidateRequest, - ProspectiveParachainsMessage, ProvisionableData, ProvisionerMessage, PvfExecution, + ProspectiveParachainsMessage, ProvisionableData, ProvisionerMessage, PvfExecutionPriority, RuntimeApiMessage, RuntimeApiRequest, StatementDistributionMessage, StoreAvailableDataError, }, @@ -632,7 +632,7 @@ async fn request_candidate_validation( candidate_receipt, pov, executor_params, - exec_kind: PvfExecution::Backing, + exec_kind: PvfExecutionPriority::Backing, response_sender: tx, }) .await; diff --git a/polkadot/node/core/backing/src/tests/mod.rs b/polkadot/node/core/backing/src/tests/mod.rs index 56d5a744c90c2..6aeca9eb7455c 100644 --- a/polkadot/node/core/backing/src/tests/mod.rs +++ b/polkadot/node/core/backing/src/tests/mod.rs @@ -405,7 +405,7 @@ async fn assert_validate_from_exhaustive( ) if validation_data == *assert_pvd && validation_code == *assert_validation_code && *pov == *assert_pov && &candidate_receipt.descriptor == assert_candidate.descriptor() && - exec_kind == PvfExecution::Backing && + exec_kind == PvfExecutionPriority::Backing && candidate_receipt.commitments_hash == assert_candidate.commitments.hash() => { response_sender.send(Ok(ValidationResult::Valid( @@ -622,7 +622,7 @@ fn backing_works(#[case] elastic_scaling_mvp: bool) { ) if validation_data == pvd_ab && validation_code == validation_code_ab && *pov == pov_ab && &candidate_receipt.descriptor == candidate_a.descriptor() && - exec_kind == PvfExecution::Backing && + exec_kind == PvfExecutionPriority::Backing && candidate_receipt.commitments_hash == candidate_a_commitments_hash => { response_sender.send(Ok( @@ -1250,7 +1250,7 @@ fn backing_works_while_validation_ongoing() { ) if validation_data == pvd_abc && validation_code == validation_code_abc && *pov == pov_abc && &candidate_receipt.descriptor == candidate_a.descriptor() && - exec_kind == PvfExecution::Backing && + exec_kind == PvfExecutionPriority::Backing && candidate_a_commitments_hash == candidate_receipt.commitments_hash => { // we never validate the candidate. our local node @@ -1417,7 +1417,7 @@ fn backing_misbehavior_works() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_a && &candidate_receipt.descriptor == candidate_a.descriptor() && - exec_kind == PvfExecution::Backing && + exec_kind == PvfExecutionPriority::Backing && candidate_a_commitments_hash == candidate_receipt.commitments_hash => { response_sender.send(Ok( @@ -1584,7 +1584,7 @@ fn backing_dont_second_invalid() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_block_a && &candidate_receipt.descriptor == candidate_a.descriptor() && - exec_kind == PvfExecution::Backing && + exec_kind == PvfExecutionPriority::Backing && candidate_a.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Ok(ValidationResult::Invalid(InvalidCandidate::BadReturn))).unwrap(); @@ -1624,7 +1624,7 @@ fn backing_dont_second_invalid() { ) if validation_data == pvd_b && validation_code == validation_code_b && *pov == pov_block_b && &candidate_receipt.descriptor == candidate_b.descriptor() && - exec_kind == PvfExecution::Backing && + exec_kind == PvfExecutionPriority::Backing && candidate_b.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Ok( @@ -1751,7 +1751,7 @@ fn backing_second_after_first_fails_works() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_a && &candidate_receipt.descriptor == candidate.descriptor() && - exec_kind == PvfExecution::Backing && + exec_kind == PvfExecutionPriority::Backing && candidate.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Ok(ValidationResult::Invalid(InvalidCandidate::BadReturn))).unwrap(); @@ -1895,7 +1895,7 @@ fn backing_works_after_failed_validation() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_a && &candidate_receipt.descriptor == candidate.descriptor() && - exec_kind == PvfExecution::Backing && + exec_kind == PvfExecutionPriority::Backing && candidate.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Err(ValidationFailed("Internal test error".into()))).unwrap(); @@ -2174,7 +2174,7 @@ fn retry_works() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_a && &candidate_receipt.descriptor == candidate.descriptor() && - exec_kind == PvfExecution::Backing && + exec_kind == PvfExecutionPriority::Backing && candidate.commitments.hash() == candidate_receipt.commitments_hash ); virtual_overseer @@ -2716,7 +2716,7 @@ fn validator_ignores_statements_from_disabled_validators() { ) if validation_data == pvd && validation_code == expected_validation_code && *pov == expected_pov && &candidate_receipt.descriptor == candidate.descriptor() && - exec_kind == PvfExecution::Backing && + exec_kind == PvfExecutionPriority::Backing && candidate_commitments_hash == candidate_receipt.commitments_hash => { response_sender.send(Ok( diff --git a/polkadot/node/core/backing/src/tests/prospective_parachains.rs b/polkadot/node/core/backing/src/tests/prospective_parachains.rs index f491f22300514..a07a8a48a8858 100644 --- a/polkadot/node/core/backing/src/tests/prospective_parachains.rs +++ b/polkadot/node/core/backing/src/tests/prospective_parachains.rs @@ -256,7 +256,7 @@ async fn assert_validate_seconded_candidate( &validation_code == assert_validation_code && &*pov == assert_pov && &candidate_receipt.descriptor == candidate.descriptor() && - exec_kind == PvfExecution::Backing && + exec_kind == PvfExecutionPriority::Backing && candidate.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Ok(ValidationResult::Valid( diff --git a/polkadot/node/core/candidate-validation/src/lib.rs b/polkadot/node/core/candidate-validation/src/lib.rs index be447c189e0b5..180356ff8a72c 100644 --- a/polkadot/node/core/candidate-validation/src/lib.rs +++ b/polkadot/node/core/candidate-validation/src/lib.rs @@ -33,7 +33,7 @@ use polkadot_node_primitives::{ use polkadot_node_subsystem::{ errors::RuntimeApiError, messages::{ - CandidateValidationMessage, PreCheckOutcome, PvfExecution, RuntimeApiMessage, + CandidateValidationMessage, PreCheckOutcome, PvfExecutionPriority, RuntimeApiMessage, RuntimeApiRequest, ValidationFailed, }, overseer, FromOrchestra, OverseerSignal, SpawnedSubsystem, SubsystemError, SubsystemResult, @@ -542,7 +542,7 @@ async fn validate_from_chain_state( candidate_receipt: CandidateReceipt, pov: Arc, executor_params: ExecutorParams, - exec_kind: PvfExecution, + exec_kind: PvfExecutionPriority, metrics: &Metrics, ) -> Result where @@ -598,7 +598,7 @@ async fn validate_candidate_exhaustive( candidate_receipt: CandidateReceipt, pov: Arc, executor_params: ExecutorParams, - exec_kind: PvfExecution, + exec_kind: PvfExecutionPriority, metrics: &Metrics, ) -> Result { let _timer = metrics.time_validate_candidate_exhaustive(); @@ -660,7 +660,7 @@ async fn validate_candidate_exhaustive( let result = match exec_kind { // Retry is disabled to reduce the chance of nondeterministic blocks getting backed and // honest backers getting slashed. - PvfExecution::Backing => { + PvfExecutionPriority::Backing => { let prep_timeout = pvf_prep_timeout(&executor_params, PvfPrepKind::Prepare); let exec_timeout = pvf_exec_timeout(&executor_params, exec_kind.into()); let pvf = PvfPrepData::from_code( @@ -680,7 +680,7 @@ async fn validate_candidate_exhaustive( ) .await }, - PvfExecution::Approval | PvfExecution::Dispute => + PvfExecutionPriority::Approval | PvfExecutionPriority::Dispute => validation_backend .validate_candidate_with_retry( raw_validation_code.to_vec(), diff --git a/polkadot/node/core/candidate-validation/src/tests.rs b/polkadot/node/core/candidate-validation/src/tests.rs index b15db1479bad5..2b7e192bf3b09 100644 --- a/polkadot/node/core/candidate-validation/src/tests.rs +++ b/polkadot/node/core/candidate-validation/src/tests.rs @@ -447,7 +447,7 @@ fn candidate_validation_ok_is_ok() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecution::Backing, + PvfExecutionPriority::Backing, &Default::default(), )) .unwrap(); @@ -499,7 +499,7 @@ fn candidate_validation_bad_return_is_invalid() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecution::Backing, + PvfExecutionPriority::Backing, &Default::default(), )) .unwrap(); @@ -580,7 +580,7 @@ fn candidate_validation_one_ambiguous_error_is_valid() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecution::Approval, + PvfExecutionPriority::Approval, &Default::default(), )) .unwrap(); @@ -621,7 +621,7 @@ fn candidate_validation_multiple_ambiguous_errors_is_invalid() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecution::Approval, + PvfExecutionPriority::Approval, &Default::default(), )) .unwrap(); @@ -633,7 +633,7 @@ fn candidate_validation_multiple_ambiguous_errors_is_invalid() { #[test] fn candidate_validation_retry_internal_errors() { let v = candidate_validation_retry_on_error_helper( - PvfExecution::Approval, + PvfExecutionPriority::Approval, vec![ Err(InternalValidationError::HostCommunication("foo".into()).into()), // Throw an AJD error, we should still retry again. @@ -651,7 +651,7 @@ fn candidate_validation_retry_internal_errors() { #[test] fn candidate_validation_dont_retry_internal_errors() { let v = candidate_validation_retry_on_error_helper( - PvfExecution::Backing, + PvfExecutionPriority::Backing, vec![ Err(InternalValidationError::HostCommunication("foo".into()).into()), // Throw an AWD error, we should still retry again. @@ -668,7 +668,7 @@ fn candidate_validation_dont_retry_internal_errors() { #[test] fn candidate_validation_retry_panic_errors() { let v = candidate_validation_retry_on_error_helper( - PvfExecution::Approval, + PvfExecutionPriority::Approval, vec![ Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::JobError("foo".into()))), // Throw an AWD error, we should still retry again. @@ -685,7 +685,7 @@ fn candidate_validation_retry_panic_errors() { #[test] fn candidate_validation_dont_retry_panic_errors() { let v = candidate_validation_retry_on_error_helper( - PvfExecution::Backing, + PvfExecutionPriority::Backing, vec![ Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::JobError("foo".into()))), // Throw an AWD error, we should still retry again. @@ -699,7 +699,7 @@ fn candidate_validation_dont_retry_panic_errors() { } fn candidate_validation_retry_on_error_helper( - exec_kind: PvfExecution, + exec_kind: PvfExecutionPriority, mock_errors: Vec>, ) -> Result { let validation_data = PersistedValidationData { max_pov_size: 1024, ..Default::default() }; @@ -777,7 +777,7 @@ fn candidate_validation_timeout_is_internal_error() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecution::Backing, + PvfExecutionPriority::Backing, &Default::default(), )); @@ -822,7 +822,7 @@ fn candidate_validation_commitment_hash_mismatch_is_invalid() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecution::Backing, + PvfExecutionPriority::Backing, &Default::default(), )) .unwrap(); @@ -874,7 +874,7 @@ fn candidate_validation_code_mismatch_is_invalid() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecution::Backing, + PvfExecutionPriority::Backing, &Default::default(), )) .unwrap(); @@ -931,7 +931,7 @@ fn compressed_code_works() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecution::Backing, + PvfExecutionPriority::Backing, &Default::default(), )); @@ -985,7 +985,7 @@ fn code_decompression_failure_is_error() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecution::Backing, + PvfExecutionPriority::Backing, &Default::default(), )); @@ -1040,7 +1040,7 @@ fn pov_decompression_failure_is_invalid() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecution::Backing, + PvfExecutionPriority::Backing, &Default::default(), )); diff --git a/polkadot/node/core/dispute-coordinator/src/participation/mod.rs b/polkadot/node/core/dispute-coordinator/src/participation/mod.rs index fbf5c24ccd162..7f0fe714659e6 100644 --- a/polkadot/node/core/dispute-coordinator/src/participation/mod.rs +++ b/polkadot/node/core/dispute-coordinator/src/participation/mod.rs @@ -27,7 +27,7 @@ use futures_timer::Delay; use polkadot_node_primitives::ValidationResult; use polkadot_node_subsystem::{ - messages::{AvailabilityRecoveryMessage, CandidateValidationMessage, PvfExecution}, + messages::{AvailabilityRecoveryMessage, CandidateValidationMessage, PvfExecutionPriority}, overseer, ActiveLeavesUpdate, RecoveryError, }; use polkadot_node_subsystem_util::runtime::get_validation_code_by_hash; @@ -385,7 +385,7 @@ async fn participate( candidate_receipt: req.candidate_receipt().clone(), pov: available_data.pov, executor_params: req.executor_params(), - exec_kind: PvfExecution::Dispute, + exec_kind: PvfExecutionPriority::Dispute, response_sender: validation_tx, }) .await; diff --git a/polkadot/node/core/dispute-coordinator/src/participation/tests.rs b/polkadot/node/core/dispute-coordinator/src/participation/tests.rs index 3999e4a9688ad..af5568a3feb04 100644 --- a/polkadot/node/core/dispute-coordinator/src/participation/tests.rs +++ b/polkadot/node/core/dispute-coordinator/src/participation/tests.rs @@ -26,8 +26,8 @@ use codec::Encode; use polkadot_node_primitives::{AvailableData, BlockData, InvalidCandidate, PoV}; use polkadot_node_subsystem::{ messages::{ - AllMessages, ChainApiMessage, DisputeCoordinatorMessage, PvfExecution, RuntimeApiMessage, - RuntimeApiRequest, + AllMessages, ChainApiMessage, DisputeCoordinatorMessage, PvfExecutionPriority, + RuntimeApiMessage, RuntimeApiRequest, }, ActiveLeavesUpdate, SpawnGlue, }; @@ -116,7 +116,7 @@ pub async fn participation_full_happy_path( ctx_handle.recv().await, AllMessages::CandidateValidation( CandidateValidationMessage::ValidateFromExhaustive { candidate_receipt, exec_kind, response_sender, .. } - ) if exec_kind == PvfExecution::Dispute => { + ) if exec_kind == PvfExecutionPriority::Dispute => { if expected_commitments_hash != candidate_receipt.commitments_hash { response_sender.send(Ok(ValidationResult::Invalid(InvalidCandidate::CommitmentsHashMismatch))).unwrap(); } else { @@ -450,7 +450,7 @@ fn cast_invalid_vote_if_validation_fails_or_is_invalid() { ctx_handle.recv().await, AllMessages::CandidateValidation( CandidateValidationMessage::ValidateFromExhaustive { exec_kind, response_sender, .. } - ) if exec_kind == PvfExecution::Dispute => { + ) if exec_kind == PvfExecutionPriority::Dispute => { response_sender.send(Ok(ValidationResult::Invalid(InvalidCandidate::Timeout))).unwrap(); }, "overseer did not receive candidate validation message", @@ -487,7 +487,7 @@ fn cast_invalid_vote_if_commitments_dont_match() { ctx_handle.recv().await, AllMessages::CandidateValidation( CandidateValidationMessage::ValidateFromExhaustive { exec_kind, response_sender, .. } - ) if exec_kind == PvfExecution::Dispute => { + ) if exec_kind == PvfExecutionPriority::Dispute => { response_sender.send(Ok(ValidationResult::Invalid(InvalidCandidate::CommitmentsHashMismatch))).unwrap(); }, "overseer did not receive candidate validation message", @@ -524,7 +524,7 @@ fn cast_valid_vote_if_validation_passes() { ctx_handle.recv().await, AllMessages::CandidateValidation( CandidateValidationMessage::ValidateFromExhaustive { exec_kind, response_sender, .. } - ) if exec_kind == PvfExecution::Dispute => { + ) if exec_kind == PvfExecutionPriority::Dispute => { response_sender.send(Ok(ValidationResult::Valid(dummy_candidate_commitments(None), PersistedValidationData::default()))).unwrap(); }, "overseer did not receive candidate validation message", diff --git a/polkadot/node/core/pvf/src/execute/queue.rs b/polkadot/node/core/pvf/src/execute/queue.rs index 3b9cd9959da61..9439e15c345fd 100644 --- a/polkadot/node/core/pvf/src/execute/queue.rs +++ b/polkadot/node/core/pvf/src/execute/queue.rs @@ -621,39 +621,45 @@ pub fn start( struct Unscheduled { unscheduled: HashMap>, - priority_log: VecDeque, + // Counter for (Low, Normal, Critical) priorities + priority_log: (u8, u8, u8), } impl Unscheduled { /// The minimum number of items with normal priority that must be present, /// regardless of the amount of critical priority items. /// In percents. - const MIN_NORMAL_PRIORITY_PRESENCE: usize = 10; + const MIN_NORMAL_PRIORITY_PRESENCE: u8 = 10; + const MAX_LOG_SIZE: u8 = 10; fn new() -> Self { Self { unscheduled: ExecutePriority::iter() .map(|priority| (priority, VecDeque::new())) .collect(), - priority_log: VecDeque::with_capacity(10), + priority_log: (0, 0, 0), } } fn log_priority(&mut self, priority: ExecutePriority) { - if self.priority_log.len() == self.priority_log.capacity() { - let _ = self.priority_log.pop_front(); + let (low, normal, critical) = &mut self.priority_log; + if (*low + *normal + *critical) > Self::MAX_LOG_SIZE { + (*low, *normal, *critical) = (0, 0, 0); + } + match priority { + ExecutePriority::Low => *low += 1, + ExecutePriority::Normal => *normal += 1, + ExecutePriority::Critical => *critical += 1, } - self.priority_log.push_back(priority); } fn select_next_priority(&self) -> ExecutePriority { - let normal_count = self.priority_log.iter().filter(|v| v.is_normal()).count(); - let normal_presense = if self.priority_log.len() == 0 { - 100 // Do critical first - } else { - normal_count * 100 / self.priority_log.len() - }; - let mut priority = if normal_presense < Self::MIN_NORMAL_PRIORITY_PRESENCE { + let (_, normal, critical) = self.priority_log; + // We do critical tasks first but ensure a minimum presence of normal tasks. + // Low priority tasks are processed only if other queues are empty. + let mut priority = if critical == 0 { + ExecutePriority::Critical // Do critical first, also avoids zero division + } else if normal * 100 / critical < Self::MIN_NORMAL_PRIORITY_PRESENCE { ExecutePriority::Normal } else { ExecutePriority::Critical diff --git a/polkadot/node/core/pvf/src/priority.rs b/polkadot/node/core/pvf/src/priority.rs index 6a7e13248496d..5190673f9479f 100644 --- a/polkadot/node/core/pvf/src/priority.rs +++ b/polkadot/node/core/pvf/src/priority.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use polkadot_node_subsystem::messages::PvfExecution; +use polkadot_node_subsystem::messages::PvfExecutionPriority; /// A priority assigned to preparation of a PVF. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] @@ -38,12 +38,12 @@ impl PreparePriority { } } -impl From for PreparePriority { - fn from(priority: PvfExecution) -> Self { +impl From for PreparePriority { + fn from(priority: PvfExecutionPriority) -> Self { match priority { - PvfExecution::Backing => PreparePriority::Normal, - PvfExecution::Approval => PreparePriority::Critical, - PvfExecution::Dispute => PreparePriority::Critical, + PvfExecutionPriority::Backing => PreparePriority::Normal, + PvfExecutionPriority::Approval => PreparePriority::Critical, + PvfExecutionPriority::Dispute => PreparePriority::Critical, } } } @@ -59,12 +59,12 @@ pub enum ExecutePriority { Critical, } -impl From for ExecutePriority { - fn from(priority: PvfExecution) -> Self { +impl From for ExecutePriority { + fn from(priority: PvfExecutionPriority) -> Self { match priority { - PvfExecution::Backing => ExecutePriority::Low, - PvfExecution::Approval => ExecutePriority::Normal, - PvfExecution::Dispute => ExecutePriority::Critical, + PvfExecutionPriority::Backing => ExecutePriority::Low, + PvfExecutionPriority::Approval => ExecutePriority::Normal, + PvfExecutionPriority::Dispute => ExecutePriority::Critical, } } } diff --git a/polkadot/node/overseer/examples/minimal-example.rs b/polkadot/node/overseer/examples/minimal-example.rs index e739311779d52..c22ca27aa585e 100644 --- a/polkadot/node/overseer/examples/minimal-example.rs +++ b/polkadot/node/overseer/examples/minimal-example.rs @@ -24,7 +24,7 @@ use orchestra::async_trait; use std::time::Duration; use polkadot_node_primitives::{BlockData, PoV}; -use polkadot_node_subsystem_types::messages::{CandidateValidationMessage, PvfExecution}; +use polkadot_node_subsystem_types::messages::{CandidateValidationMessage, PvfExecutionPriority}; use polkadot_overseer::{ self as overseer, dummy::dummy_overseer_builder, @@ -77,7 +77,7 @@ impl Subsystem1 { candidate_receipt, pov: PoV { block_data: BlockData(Vec::new()) }.into(), executor_params: Default::default(), - exec_kind: PvfExecution::Backing, + exec_kind: PvfExecutionPriority::Backing, response_sender: tx, }; ctx.send_message(msg).await; diff --git a/polkadot/node/overseer/src/tests.rs b/polkadot/node/overseer/src/tests.rs index 506501a2e7075..d60bd53ba0564 100644 --- a/polkadot/node/overseer/src/tests.rs +++ b/polkadot/node/overseer/src/tests.rs @@ -25,7 +25,7 @@ use polkadot_node_primitives::{ }; use polkadot_node_subsystem_test_helpers::mock::{dummy_unpin_handle, new_leaf}; use polkadot_node_subsystem_types::messages::{ - NetworkBridgeEvent, PvfExecution, ReportPeerMessage, RuntimeApiRequest, + NetworkBridgeEvent, PvfExecutionPriority, ReportPeerMessage, RuntimeApiRequest, }; use polkadot_primitives::{ CandidateHash, CandidateReceipt, CollatorPair, Id as ParaId, InvalidDisputeStatementKind, @@ -108,7 +108,7 @@ where candidate_receipt, pov: PoV { block_data: BlockData(Vec::new()) }.into(), executor_params: Default::default(), - exec_kind: PvfExecution::Backing, + exec_kind: PvfExecutionPriority::Backing, response_sender: tx, }) .await; @@ -806,7 +806,7 @@ fn test_candidate_validation_msg() -> CandidateValidationMessage { candidate_receipt, pov, executor_params: Default::default(), - exec_kind: PvfExecution::Backing, + exec_kind: PvfExecutionPriority::Backing, response_sender, } } diff --git a/polkadot/node/subsystem-types/src/messages.rs b/polkadot/node/subsystem-types/src/messages.rs index f0a9cceab9e00..7a578cec17ea0 100644 --- a/polkadot/node/subsystem-types/src/messages.rs +++ b/polkadot/node/subsystem-types/src/messages.rs @@ -160,7 +160,7 @@ pub enum CandidateValidationMessage { /// Session's executor parameters executor_params: ExecutorParams, /// Execution kind, used for timeouts and retries (backing/approvals) - exec_kind: PvfExecution, + exec_kind: PvfExecutionPriority, /// The sending side of the response channel response_sender: oneshot::Sender>, }, @@ -185,7 +185,7 @@ pub enum CandidateValidationMessage { /// Session's executor parameters executor_params: ExecutorParams, /// Execution kind, used for timeouts and retries (backing/approvals) - exec_kind: PvfExecution, + exec_kind: PvfExecutionPriority, /// The sending side of the response channel response_sender: oneshot::Sender>, }, @@ -207,7 +207,7 @@ pub enum CandidateValidationMessage { /// Extends primitives::PvfExecKind to have a separate value for duspute requests /// which is important for prioritization. #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum PvfExecution { +pub enum PvfExecutionPriority { /// For backing requests. Backing, /// For approval requests @@ -216,12 +216,12 @@ pub enum PvfExecution { Dispute, } -impl From for PvfExecKind { - fn from(exec: PvfExecution) -> Self { +impl From for PvfExecKind { + fn from(exec: PvfExecutionPriority) -> Self { match exec { - PvfExecution::Backing => PvfExecKind::Backing, - PvfExecution::Approval => PvfExecKind::Approval, - PvfExecution::Dispute => PvfExecKind::Approval, + PvfExecutionPriority::Backing => PvfExecKind::Backing, + PvfExecutionPriority::Approval => PvfExecKind::Approval, + PvfExecutionPriority::Dispute => PvfExecKind::Approval, } } } From 0c1edcf49c4a94b841978a0cd0e7cbdf594bbb5e Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Wed, 10 Jul 2024 14:12:13 +0200 Subject: [PATCH 13/48] Update logic --- polkadot/node/core/approval-voting/src/lib.rs | 4 +- .../node/core/approval-voting/src/tests.rs | 10 +- polkadot/node/core/backing/src/lib.rs | 4 +- polkadot/node/core/backing/src/tests/mod.rs | 20 +- .../src/tests/prospective_parachains.rs | 2 +- .../node/core/candidate-validation/src/lib.rs | 10 +- .../core/candidate-validation/src/tests.rs | 30 +-- .../src/participation/mod.rs | 4 +- .../src/participation/tests.rs | 10 +- polkadot/node/core/pvf/src/execute/queue.rs | 207 ++++++++++-------- polkadot/node/core/pvf/src/host.rs | 32 +-- polkadot/node/core/pvf/src/lib.rs | 2 +- polkadot/node/core/pvf/src/priority.rs | 55 ----- .../node/overseer/examples/minimal-example.rs | 4 +- polkadot/node/overseer/src/tests.rs | 6 +- polkadot/node/subsystem-types/src/messages.rs | 48 ++-- 16 files changed, 222 insertions(+), 226 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index de966f3ddb0c8..4b0405ebf6057 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -41,7 +41,7 @@ use polkadot_node_subsystem::{ ApprovalVotingMessage, AssignmentCheckError, AssignmentCheckResult, AvailabilityRecoveryMessage, BlockDescription, CandidateValidationMessage, ChainApiMessage, ChainSelectionMessage, DisputeCoordinatorMessage, HighestApprovedAncestorBlock, - PvfExecutionPriority, RuntimeApiMessage, RuntimeApiRequest, + PvfExecPriority, RuntimeApiMessage, RuntimeApiRequest, }, overseer, FromOrchestra, OverseerSignal, SpawnedSubsystem, SubsystemError, SubsystemResult, SubsystemSender, @@ -3529,7 +3529,7 @@ async fn launch_approval( candidate_receipt: candidate.clone(), pov: available_data.pov, executor_params, - exec_kind: PvfExecutionPriority::Approval, + exec_kind: PvfExecPriority::Approval, response_sender: val_tx, }) .await; diff --git a/polkadot/node/core/approval-voting/src/tests.rs b/polkadot/node/core/approval-voting/src/tests.rs index 766ed6fb5cc5f..533e04079eb79 100644 --- a/polkadot/node/core/approval-voting/src/tests.rs +++ b/polkadot/node/core/approval-voting/src/tests.rs @@ -3270,7 +3270,7 @@ async fn handle_double_assignment_import( exec_kind, response_sender, .. - }) if exec_kind == PvfExecutionPriority::Approval => { + }) if exec_kind == PvfExecPriority::Approval => { response_sender.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) .unwrap(); } @@ -4210,7 +4210,7 @@ async fn handle_approval_on_max_coalesce_count( for _ in &candidate_indices { assert_matches!( overseer_recv(virtual_overseer).await, - AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive{exec_kind, response_sender, ..}) if exec_kind == PvfExecutionPriority::Approval => { + AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive{exec_kind, response_sender, ..}) if exec_kind == PvfExecPriority::Approval => { response_sender.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) .unwrap(); } @@ -4274,7 +4274,7 @@ async fn handle_approval_on_max_wait_time( for _ in &candidate_indices { assert_matches!( overseer_recv(virtual_overseer).await, - AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive{exec_kind, response_sender, ..}) if exec_kind == PvfExecutionPriority::Approval => { + AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive{exec_kind, response_sender, ..}) if exec_kind == PvfExecPriority::Approval => { response_sender.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) .unwrap(); } @@ -4777,7 +4777,7 @@ fn subsystem_relaunches_approval_work_on_restart() { exec_kind, response_sender, .. - }) if exec_kind == PvfExecutionPriority::Approval => { + }) if exec_kind == PvfExecPriority::Approval => { response_sender.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) .unwrap(); } @@ -4897,7 +4897,7 @@ fn subsystem_sends_pending_approvals_on_approval_restart() { exec_kind, response_sender, .. - }) if exec_kind == PvfExecutionPriority::Approval => { + }) if exec_kind == PvfExecPriority::Approval => { response_sender.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) .unwrap(); } diff --git a/polkadot/node/core/backing/src/lib.rs b/polkadot/node/core/backing/src/lib.rs index fd8d6d8fedbbf..46cf9696161ab 100644 --- a/polkadot/node/core/backing/src/lib.rs +++ b/polkadot/node/core/backing/src/lib.rs @@ -89,7 +89,7 @@ use polkadot_node_subsystem::{ AvailabilityDistributionMessage, AvailabilityStoreMessage, CanSecondRequest, CandidateBackingMessage, CandidateValidationMessage, CollatorProtocolMessage, HypotheticalCandidate, HypotheticalMembershipRequest, IntroduceSecondedCandidateRequest, - ProspectiveParachainsMessage, ProvisionableData, ProvisionerMessage, PvfExecutionPriority, + ProspectiveParachainsMessage, ProvisionableData, ProvisionerMessage, PvfExecPriority, RuntimeApiMessage, RuntimeApiRequest, StatementDistributionMessage, StoreAvailableDataError, }, @@ -632,7 +632,7 @@ async fn request_candidate_validation( candidate_receipt, pov, executor_params, - exec_kind: PvfExecutionPriority::Backing, + exec_kind: PvfExecPriority::Backing, response_sender: tx, }) .await; diff --git a/polkadot/node/core/backing/src/tests/mod.rs b/polkadot/node/core/backing/src/tests/mod.rs index 6aeca9eb7455c..0378357649f5f 100644 --- a/polkadot/node/core/backing/src/tests/mod.rs +++ b/polkadot/node/core/backing/src/tests/mod.rs @@ -405,7 +405,7 @@ async fn assert_validate_from_exhaustive( ) if validation_data == *assert_pvd && validation_code == *assert_validation_code && *pov == *assert_pov && &candidate_receipt.descriptor == assert_candidate.descriptor() && - exec_kind == PvfExecutionPriority::Backing && + exec_kind == PvfExecPriority::Backing && candidate_receipt.commitments_hash == assert_candidate.commitments.hash() => { response_sender.send(Ok(ValidationResult::Valid( @@ -622,7 +622,7 @@ fn backing_works(#[case] elastic_scaling_mvp: bool) { ) if validation_data == pvd_ab && validation_code == validation_code_ab && *pov == pov_ab && &candidate_receipt.descriptor == candidate_a.descriptor() && - exec_kind == PvfExecutionPriority::Backing && + exec_kind == PvfExecPriority::Backing && candidate_receipt.commitments_hash == candidate_a_commitments_hash => { response_sender.send(Ok( @@ -1250,7 +1250,7 @@ fn backing_works_while_validation_ongoing() { ) if validation_data == pvd_abc && validation_code == validation_code_abc && *pov == pov_abc && &candidate_receipt.descriptor == candidate_a.descriptor() && - exec_kind == PvfExecutionPriority::Backing && + exec_kind == PvfExecPriority::Backing && candidate_a_commitments_hash == candidate_receipt.commitments_hash => { // we never validate the candidate. our local node @@ -1417,7 +1417,7 @@ fn backing_misbehavior_works() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_a && &candidate_receipt.descriptor == candidate_a.descriptor() && - exec_kind == PvfExecutionPriority::Backing && + exec_kind == PvfExecPriority::Backing && candidate_a_commitments_hash == candidate_receipt.commitments_hash => { response_sender.send(Ok( @@ -1584,7 +1584,7 @@ fn backing_dont_second_invalid() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_block_a && &candidate_receipt.descriptor == candidate_a.descriptor() && - exec_kind == PvfExecutionPriority::Backing && + exec_kind == PvfExecPriority::Backing && candidate_a.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Ok(ValidationResult::Invalid(InvalidCandidate::BadReturn))).unwrap(); @@ -1624,7 +1624,7 @@ fn backing_dont_second_invalid() { ) if validation_data == pvd_b && validation_code == validation_code_b && *pov == pov_block_b && &candidate_receipt.descriptor == candidate_b.descriptor() && - exec_kind == PvfExecutionPriority::Backing && + exec_kind == PvfExecPriority::Backing && candidate_b.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Ok( @@ -1751,7 +1751,7 @@ fn backing_second_after_first_fails_works() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_a && &candidate_receipt.descriptor == candidate.descriptor() && - exec_kind == PvfExecutionPriority::Backing && + exec_kind == PvfExecPriority::Backing && candidate.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Ok(ValidationResult::Invalid(InvalidCandidate::BadReturn))).unwrap(); @@ -1895,7 +1895,7 @@ fn backing_works_after_failed_validation() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_a && &candidate_receipt.descriptor == candidate.descriptor() && - exec_kind == PvfExecutionPriority::Backing && + exec_kind == PvfExecPriority::Backing && candidate.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Err(ValidationFailed("Internal test error".into()))).unwrap(); @@ -2174,7 +2174,7 @@ fn retry_works() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_a && &candidate_receipt.descriptor == candidate.descriptor() && - exec_kind == PvfExecutionPriority::Backing && + exec_kind == PvfExecPriority::Backing && candidate.commitments.hash() == candidate_receipt.commitments_hash ); virtual_overseer @@ -2716,7 +2716,7 @@ fn validator_ignores_statements_from_disabled_validators() { ) if validation_data == pvd && validation_code == expected_validation_code && *pov == expected_pov && &candidate_receipt.descriptor == candidate.descriptor() && - exec_kind == PvfExecutionPriority::Backing && + exec_kind == PvfExecPriority::Backing && candidate_commitments_hash == candidate_receipt.commitments_hash => { response_sender.send(Ok( diff --git a/polkadot/node/core/backing/src/tests/prospective_parachains.rs b/polkadot/node/core/backing/src/tests/prospective_parachains.rs index a07a8a48a8858..f3aff6ba96be6 100644 --- a/polkadot/node/core/backing/src/tests/prospective_parachains.rs +++ b/polkadot/node/core/backing/src/tests/prospective_parachains.rs @@ -256,7 +256,7 @@ async fn assert_validate_seconded_candidate( &validation_code == assert_validation_code && &*pov == assert_pov && &candidate_receipt.descriptor == candidate.descriptor() && - exec_kind == PvfExecutionPriority::Backing && + exec_kind == PvfExecPriority::Backing && candidate.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Ok(ValidationResult::Valid( diff --git a/polkadot/node/core/candidate-validation/src/lib.rs b/polkadot/node/core/candidate-validation/src/lib.rs index 180356ff8a72c..1d9b641ca8798 100644 --- a/polkadot/node/core/candidate-validation/src/lib.rs +++ b/polkadot/node/core/candidate-validation/src/lib.rs @@ -33,7 +33,7 @@ use polkadot_node_primitives::{ use polkadot_node_subsystem::{ errors::RuntimeApiError, messages::{ - CandidateValidationMessage, PreCheckOutcome, PvfExecutionPriority, RuntimeApiMessage, + CandidateValidationMessage, PreCheckOutcome, PvfExecPriority, RuntimeApiMessage, RuntimeApiRequest, ValidationFailed, }, overseer, FromOrchestra, OverseerSignal, SpawnedSubsystem, SubsystemError, SubsystemResult, @@ -542,7 +542,7 @@ async fn validate_from_chain_state( candidate_receipt: CandidateReceipt, pov: Arc, executor_params: ExecutorParams, - exec_kind: PvfExecutionPriority, + exec_kind: PvfExecPriority, metrics: &Metrics, ) -> Result where @@ -598,7 +598,7 @@ async fn validate_candidate_exhaustive( candidate_receipt: CandidateReceipt, pov: Arc, executor_params: ExecutorParams, - exec_kind: PvfExecutionPriority, + exec_kind: PvfExecPriority, metrics: &Metrics, ) -> Result { let _timer = metrics.time_validate_candidate_exhaustive(); @@ -660,7 +660,7 @@ async fn validate_candidate_exhaustive( let result = match exec_kind { // Retry is disabled to reduce the chance of nondeterministic blocks getting backed and // honest backers getting slashed. - PvfExecutionPriority::Backing => { + PvfExecPriority::Backing => { let prep_timeout = pvf_prep_timeout(&executor_params, PvfPrepKind::Prepare); let exec_timeout = pvf_exec_timeout(&executor_params, exec_kind.into()); let pvf = PvfPrepData::from_code( @@ -680,7 +680,7 @@ async fn validate_candidate_exhaustive( ) .await }, - PvfExecutionPriority::Approval | PvfExecutionPriority::Dispute => + PvfExecPriority::Approval | PvfExecPriority::Dispute => validation_backend .validate_candidate_with_retry( raw_validation_code.to_vec(), diff --git a/polkadot/node/core/candidate-validation/src/tests.rs b/polkadot/node/core/candidate-validation/src/tests.rs index 2b7e192bf3b09..887c37ac6b878 100644 --- a/polkadot/node/core/candidate-validation/src/tests.rs +++ b/polkadot/node/core/candidate-validation/src/tests.rs @@ -447,7 +447,7 @@ fn candidate_validation_ok_is_ok() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecutionPriority::Backing, + PvfExecPriority::Backing, &Default::default(), )) .unwrap(); @@ -499,7 +499,7 @@ fn candidate_validation_bad_return_is_invalid() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecutionPriority::Backing, + PvfExecPriority::Backing, &Default::default(), )) .unwrap(); @@ -580,7 +580,7 @@ fn candidate_validation_one_ambiguous_error_is_valid() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecutionPriority::Approval, + PvfExecPriority::Approval, &Default::default(), )) .unwrap(); @@ -621,7 +621,7 @@ fn candidate_validation_multiple_ambiguous_errors_is_invalid() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecutionPriority::Approval, + PvfExecPriority::Approval, &Default::default(), )) .unwrap(); @@ -633,7 +633,7 @@ fn candidate_validation_multiple_ambiguous_errors_is_invalid() { #[test] fn candidate_validation_retry_internal_errors() { let v = candidate_validation_retry_on_error_helper( - PvfExecutionPriority::Approval, + PvfExecPriority::Approval, vec![ Err(InternalValidationError::HostCommunication("foo".into()).into()), // Throw an AJD error, we should still retry again. @@ -651,7 +651,7 @@ fn candidate_validation_retry_internal_errors() { #[test] fn candidate_validation_dont_retry_internal_errors() { let v = candidate_validation_retry_on_error_helper( - PvfExecutionPriority::Backing, + PvfExecPriority::Backing, vec![ Err(InternalValidationError::HostCommunication("foo".into()).into()), // Throw an AWD error, we should still retry again. @@ -668,7 +668,7 @@ fn candidate_validation_dont_retry_internal_errors() { #[test] fn candidate_validation_retry_panic_errors() { let v = candidate_validation_retry_on_error_helper( - PvfExecutionPriority::Approval, + PvfExecPriority::Approval, vec![ Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::JobError("foo".into()))), // Throw an AWD error, we should still retry again. @@ -685,7 +685,7 @@ fn candidate_validation_retry_panic_errors() { #[test] fn candidate_validation_dont_retry_panic_errors() { let v = candidate_validation_retry_on_error_helper( - PvfExecutionPriority::Backing, + PvfExecPriority::Backing, vec![ Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::JobError("foo".into()))), // Throw an AWD error, we should still retry again. @@ -699,7 +699,7 @@ fn candidate_validation_dont_retry_panic_errors() { } fn candidate_validation_retry_on_error_helper( - exec_kind: PvfExecutionPriority, + exec_kind: PvfExecPriority, mock_errors: Vec>, ) -> Result { let validation_data = PersistedValidationData { max_pov_size: 1024, ..Default::default() }; @@ -777,7 +777,7 @@ fn candidate_validation_timeout_is_internal_error() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecutionPriority::Backing, + PvfExecPriority::Backing, &Default::default(), )); @@ -822,7 +822,7 @@ fn candidate_validation_commitment_hash_mismatch_is_invalid() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecutionPriority::Backing, + PvfExecPriority::Backing, &Default::default(), )) .unwrap(); @@ -874,7 +874,7 @@ fn candidate_validation_code_mismatch_is_invalid() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecutionPriority::Backing, + PvfExecPriority::Backing, &Default::default(), )) .unwrap(); @@ -931,7 +931,7 @@ fn compressed_code_works() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecutionPriority::Backing, + PvfExecPriority::Backing, &Default::default(), )); @@ -985,7 +985,7 @@ fn code_decompression_failure_is_error() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecutionPriority::Backing, + PvfExecPriority::Backing, &Default::default(), )); @@ -1040,7 +1040,7 @@ fn pov_decompression_failure_is_invalid() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecutionPriority::Backing, + PvfExecPriority::Backing, &Default::default(), )); diff --git a/polkadot/node/core/dispute-coordinator/src/participation/mod.rs b/polkadot/node/core/dispute-coordinator/src/participation/mod.rs index 7f0fe714659e6..ccb7c8e42a8bd 100644 --- a/polkadot/node/core/dispute-coordinator/src/participation/mod.rs +++ b/polkadot/node/core/dispute-coordinator/src/participation/mod.rs @@ -27,7 +27,7 @@ use futures_timer::Delay; use polkadot_node_primitives::ValidationResult; use polkadot_node_subsystem::{ - messages::{AvailabilityRecoveryMessage, CandidateValidationMessage, PvfExecutionPriority}, + messages::{AvailabilityRecoveryMessage, CandidateValidationMessage, PvfExecPriority}, overseer, ActiveLeavesUpdate, RecoveryError, }; use polkadot_node_subsystem_util::runtime::get_validation_code_by_hash; @@ -385,7 +385,7 @@ async fn participate( candidate_receipt: req.candidate_receipt().clone(), pov: available_data.pov, executor_params: req.executor_params(), - exec_kind: PvfExecutionPriority::Dispute, + exec_kind: PvfExecPriority::Dispute, response_sender: validation_tx, }) .await; diff --git a/polkadot/node/core/dispute-coordinator/src/participation/tests.rs b/polkadot/node/core/dispute-coordinator/src/participation/tests.rs index af5568a3feb04..0a2c46c31f1d5 100644 --- a/polkadot/node/core/dispute-coordinator/src/participation/tests.rs +++ b/polkadot/node/core/dispute-coordinator/src/participation/tests.rs @@ -26,7 +26,7 @@ use codec::Encode; use polkadot_node_primitives::{AvailableData, BlockData, InvalidCandidate, PoV}; use polkadot_node_subsystem::{ messages::{ - AllMessages, ChainApiMessage, DisputeCoordinatorMessage, PvfExecutionPriority, + AllMessages, ChainApiMessage, DisputeCoordinatorMessage, PvfExecPriority, RuntimeApiMessage, RuntimeApiRequest, }, ActiveLeavesUpdate, SpawnGlue, @@ -116,7 +116,7 @@ pub async fn participation_full_happy_path( ctx_handle.recv().await, AllMessages::CandidateValidation( CandidateValidationMessage::ValidateFromExhaustive { candidate_receipt, exec_kind, response_sender, .. } - ) if exec_kind == PvfExecutionPriority::Dispute => { + ) if exec_kind == PvfExecPriority::Dispute => { if expected_commitments_hash != candidate_receipt.commitments_hash { response_sender.send(Ok(ValidationResult::Invalid(InvalidCandidate::CommitmentsHashMismatch))).unwrap(); } else { @@ -450,7 +450,7 @@ fn cast_invalid_vote_if_validation_fails_or_is_invalid() { ctx_handle.recv().await, AllMessages::CandidateValidation( CandidateValidationMessage::ValidateFromExhaustive { exec_kind, response_sender, .. } - ) if exec_kind == PvfExecutionPriority::Dispute => { + ) if exec_kind == PvfExecPriority::Dispute => { response_sender.send(Ok(ValidationResult::Invalid(InvalidCandidate::Timeout))).unwrap(); }, "overseer did not receive candidate validation message", @@ -487,7 +487,7 @@ fn cast_invalid_vote_if_commitments_dont_match() { ctx_handle.recv().await, AllMessages::CandidateValidation( CandidateValidationMessage::ValidateFromExhaustive { exec_kind, response_sender, .. } - ) if exec_kind == PvfExecutionPriority::Dispute => { + ) if exec_kind == PvfExecPriority::Dispute => { response_sender.send(Ok(ValidationResult::Invalid(InvalidCandidate::CommitmentsHashMismatch))).unwrap(); }, "overseer did not receive candidate validation message", @@ -524,7 +524,7 @@ fn cast_valid_vote_if_validation_passes() { ctx_handle.recv().await, AllMessages::CandidateValidation( CandidateValidationMessage::ValidateFromExhaustive { exec_kind, response_sender, .. } - ) if exec_kind == PvfExecutionPriority::Dispute => { + ) if exec_kind == PvfExecPriority::Dispute => { response_sender.send(Ok(ValidationResult::Valid(dummy_candidate_commitments(None), PersistedValidationData::default()))).unwrap(); }, "overseer did not receive candidate validation message", diff --git a/polkadot/node/core/pvf/src/execute/queue.rs b/polkadot/node/core/pvf/src/execute/queue.rs index 9439e15c345fd..464e015c5220d 100644 --- a/polkadot/node/core/pvf/src/execute/queue.rs +++ b/polkadot/node/core/pvf/src/execute/queue.rs @@ -22,7 +22,7 @@ use crate::{ host::ResultSender, metrics::Metrics, worker_interface::{IdleWorker, WorkerHandle}, - ExecutePriority, InvalidCandidate, PossiblyInvalidError, ValidationError, LOG_TARGET, + InvalidCandidate, PossiblyInvalidError, ValidationError, LOG_TARGET, }; use futures::{ channel::{mpsc, oneshot}, @@ -34,7 +34,9 @@ use polkadot_node_core_pvf_common::{ execute::{JobResponse, WorkerError, WorkerResponse}, SecurityStatus, }; +use polkadot_node_subsystem::messages::PvfExecPriority; use polkadot_primitives::{ExecutorParams, ExecutorParamsHash}; +use rand::Rng; use slotmap::HopSlotMap; use std::{ collections::{HashMap, VecDeque}, @@ -71,7 +73,7 @@ pub struct PendingExecutionRequest { pub params: Vec, pub executor_params: ExecutorParams, pub result_tx: ResultSender, - pub execute_priority: ExecutePriority, + pub execute_priority: PvfExecPriority, } struct ExecuteJob { @@ -226,7 +228,8 @@ impl Queue { // New jobs are always pushed to the tail of the queue; the one at its head is always // the eldest one. - let Some((queue, priority)) = self.unscheduled.get_mut() else { return }; + let priority = self.unscheduled.select_next_priority(is_probable); + let Some(queue) = self.unscheduled.get_mut(priority) else { return }; let eldest = if let Some(eldest) = queue.get(0) { eldest } else { return }; @@ -277,7 +280,6 @@ impl Queue { } else { spawn_extra_worker(self, job); } - self.unscheduled.log_priority(priority); } } @@ -620,67 +622,88 @@ pub fn start( } struct Unscheduled { - unscheduled: HashMap>, - // Counter for (Low, Normal, Critical) priorities - priority_log: (u8, u8, u8), + unscheduled: HashMap>, +} + +fn is_probable(probability: u8) -> bool { + let mut rng = rand::thread_rng(); + let n: u8 = rng.gen_range(0..100); + n <= probability } impl Unscheduled { - /// The minimum number of items with normal priority that must be present, - /// regardless of the amount of critical priority items. - /// In percents. - const MIN_NORMAL_PRIORITY_PRESENCE: u8 = 10; - const MAX_LOG_SIZE: u8 = 10; + // A threshold reaching which we prioritize approval jobs + const REGULAR_JOBS_THRESHOLD: usize = 12; fn new() -> Self { Self { - unscheduled: ExecutePriority::iter() + unscheduled: PvfExecPriority::iter() .map(|priority| (priority, VecDeque::new())) .collect(), - priority_log: (0, 0, 0), } } - fn log_priority(&mut self, priority: ExecutePriority) { - let (low, normal, critical) = &mut self.priority_log; - if (*low + *normal + *critical) > Self::MAX_LOG_SIZE { - (*low, *normal, *critical) = (0, 0, 0); - } - match priority { - ExecutePriority::Low => *low += 1, - ExecutePriority::Normal => *normal += 1, - ExecutePriority::Critical => *critical += 1, + fn select_next_priority(&self, is_probable: F) -> PvfExecPriority + where + F: Fn(u8) -> bool, + { + let mut priority = self.next_priority(is_probable); + + while !self.has_pending(priority) { + let Some(p) = priority.lower() else { break }; + priority = p; } + + priority } - fn select_next_priority(&self) -> ExecutePriority { - let (_, normal, critical) = self.priority_log; - // We do critical tasks first but ensure a minimum presence of normal tasks. - // Low priority tasks are processed only if other queues are empty. - let mut priority = if critical == 0 { - ExecutePriority::Critical // Do critical first, also avoids zero division - } else if normal * 100 / critical < Self::MIN_NORMAL_PRIORITY_PRESENCE { - ExecutePriority::Normal - } else { - ExecutePriority::Critical - }; + fn next_priority(&self, is_probable: F) -> PvfExecPriority + where + F: Fn(u8) -> bool, + { + use PvfExecPriority::*; - while self.unscheduled.get(&priority).map_or(true, |v| v.is_empty()) { - let Some(lower) = priority.lower() else { break }; - priority = lower; + if self.has_pending(Dispute) && is_probable(70) { + return Dispute } - priority + if !self.has_pending(Backing) && !self.has_pending(BackingSystem) { + return Approval + } + + if self.regular_jobs_count() > Self::REGULAR_JOBS_THRESHOLD && is_probable(87) { + return Approval + } + + if is_probable(75) { + return BackingSystem + } + + return Backing } - fn get_mut(&mut self) -> Option<(&mut VecDeque, ExecutePriority)> { - let priority = self.select_next_priority(); - self.unscheduled.get_mut(&priority).map(|jobs| (jobs, priority)) + // Number of jobs in all queues excluding disputes + fn regular_jobs_count(&self) -> usize { + let mut count = 0; + for (priority, queue) in self.unscheduled.iter() { + if *priority != PvfExecPriority::Dispute { + count += queue.len(); + } + } + count } - fn add(&mut self, job: ExecuteJob, priority: ExecutePriority) { + fn get_mut(&mut self, priority: PvfExecPriority) -> Option<&mut VecDeque> { + self.unscheduled.get_mut(&priority) + } + + fn add(&mut self, job: ExecuteJob, priority: PvfExecPriority) { self.unscheduled.entry(priority).or_default().push_back(job); } + + fn has_pending(&self, kind: PvfExecPriority) -> bool { + !self.unscheduled.get(&kind).unwrap_or(&VecDeque::new()).is_empty() + } } #[cfg(test)] @@ -705,70 +728,76 @@ mod tests { fn test_unscheduled_add() { let mut unscheduled = Unscheduled::new(); - unscheduled.add(create_execution_job(), ExecutePriority::Low); - unscheduled.add(create_execution_job(), ExecutePriority::Normal); - unscheduled.add(create_execution_job(), ExecutePriority::Critical); + PvfExecPriority::iter().for_each(|priority| { + unscheduled.add(create_execution_job(), priority); + }); - let low = unscheduled.unscheduled.get(&ExecutePriority::Low).unwrap(); - let normal = unscheduled.unscheduled.get(&ExecutePriority::Normal).unwrap(); - let critical = unscheduled.unscheduled.get(&ExecutePriority::Critical).unwrap(); - - assert_eq!(low.len(), 1); - assert_eq!(normal.len(), 1); - assert_eq!(critical.len(), 1); + PvfExecPriority::iter().for_each(|priority| { + let queue = unscheduled.unscheduled.get(&priority).unwrap(); + assert_eq!(queue.len(), 1); + }); } #[test] - fn test_log_priority() { + fn test_unscheduled_select_next_priority() { + use PvfExecPriority::*; + + // With disputes let mut unscheduled = Unscheduled::new(); + unscheduled.add(create_execution_job(), Approval); + unscheduled.add(create_execution_job(), Backing); + unscheduled.add(create_execution_job(), BackingSystem); + unscheduled.add(create_execution_job(), Dispute); - for _ in 0..15 { - unscheduled.log_priority(ExecutePriority::Normal); - } + // Do disputes with probability 70% + assert_eq!(unscheduled.select_next_priority(|p| p == 70), Dispute); + assert_eq!(unscheduled.select_next_priority(|_| false), Backing); - assert_eq!(unscheduled.priority_log.len(), 10); - assert!(unscheduled.priority_log.iter().all(|&p| p == ExecutePriority::Normal)); - } + // Without disputes and backing + unscheduled.get_mut(Dispute).unwrap().clear(); + unscheduled.get_mut(BackingSystem).unwrap().clear(); + unscheduled.get_mut(Backing).unwrap().clear(); - #[test] - fn test_unscheduled_priority_selection() { - let mut unscheduled = Unscheduled::new(); + // Do approval + assert_eq!(unscheduled.select_next_priority(|_| false), Approval); - unscheduled.add(create_execution_job(), ExecutePriority::Low); + // But if no approvals + unscheduled.get_mut(Approval).unwrap().clear(); - let (queue, priority) = unscheduled.get_mut().unwrap(); + // Fallback to Backing as the lowest value + assert_eq!(unscheduled.select_next_priority(|_| true), Backing); - // Returns low priority queue if no others - assert_eq!(priority, ExecutePriority::Low); - assert_eq!(queue.len(), 1); - - for _ in 0..2 { - unscheduled.add(create_execution_job(), ExecutePriority::Normal); + // With long queue + for _ in 0..Unscheduled::REGULAR_JOBS_THRESHOLD { + unscheduled.add(create_execution_job(), Approval); } + unscheduled.add(create_execution_job(), BackingSystem); + unscheduled.add(create_execution_job(), Backing); - let (queue, priority) = unscheduled.get_mut().unwrap(); - - // Returns normal priority queue - assert_eq!(priority, ExecutePriority::Normal); - assert_eq!(queue.len(), 2); + // Do approval with probability 87% + assert_eq!(unscheduled.select_next_priority(|p| p == 87), Approval); + assert_eq!(unscheduled.select_next_priority(|_| false), Backing); - for _ in 0..3 { - unscheduled.add(create_execution_job(), ExecutePriority::Critical); + // But if no approvals + unscheduled.get_mut(Approval).unwrap().clear(); + for _ in 0..Unscheduled::REGULAR_JOBS_THRESHOLD { + unscheduled.add(create_execution_job(), BackingSystem); } - for _ in 0..10 { - unscheduled.log_priority(ExecutePriority::Critical); - } - let (queue, priority) = unscheduled.get_mut().unwrap(); - // Returns normal priority queue because logged too many criticals - assert_eq!(priority, ExecutePriority::Normal); - assert_eq!(queue.len(), 2); + // Fallback to next not empty queue priority + assert_eq!(unscheduled.select_next_priority(|_| true), BackingSystem); + + // Otherwise + unscheduled.get_mut(Approval).unwrap().clear(); + + // Do system parachains backing with probability 75% + assert_eq!(unscheduled.select_next_priority(|p| p == 75), BackingSystem); + assert_eq!(unscheduled.select_next_priority(|_| false), Backing); - unscheduled.log_priority(ExecutePriority::Normal); - let (queue, priority) = unscheduled.get_mut().unwrap(); + // But if no system parachains backing + unscheduled.get_mut(BackingSystem).unwrap().clear(); - // Returns critical priority queue because logged enough normals - assert_eq!(priority, ExecutePriority::Critical); - assert_eq!(queue.len(), 3); + // Fallback to next not empty queue priority + assert_eq!(unscheduled.select_next_priority(|_| true), Backing); } } diff --git a/polkadot/node/core/pvf/src/host.rs b/polkadot/node/core/pvf/src/host.rs index c4cc9172d7dde..78c8d67c451b3 100644 --- a/polkadot/node/core/pvf/src/host.rs +++ b/polkadot/node/core/pvf/src/host.rs @@ -24,7 +24,7 @@ use crate::{ artifacts::{ArtifactId, ArtifactPathId, ArtifactState, Artifacts, ArtifactsCleanupConfig}, execute::{self, PendingExecutionRequest}, metrics::Metrics, - prepare, ExecutePriority, PreparePriority, SecurityStatus, ValidationError, LOG_TARGET, + prepare, PreparePriority, SecurityStatus, ValidationError, LOG_TARGET, }; use always_assert::never; use futures::{ @@ -36,7 +36,7 @@ use polkadot_node_core_pvf_common::{ prepare::PrepareSuccess, pvf::PvfPrepData, }; -use polkadot_node_subsystem::{SubsystemError, SubsystemResult}; +use polkadot_node_subsystem::{messages::PvfExecPriority, SubsystemError, SubsystemResult}; use polkadot_parachain_primitives::primitives::ValidationResult; use std::{ collections::HashMap, @@ -110,7 +110,7 @@ impl ValidationHost { exec_timeout: Duration, params: Vec, priority: PreparePriority, - execute_priority: ExecutePriority, + execute_priority: PvfExecPriority, result_tx: ResultSender, ) -> Result<(), String> { self.to_host_tx @@ -151,7 +151,7 @@ struct ExecutePvfInputs { exec_timeout: Duration, params: Vec, priority: PreparePriority, - execute_priority: ExecutePriority, + execute_priority: PvfExecPriority, result_tx: ResultSender, } @@ -1258,7 +1258,7 @@ pub(crate) mod tests { TEST_EXECUTION_TIMEOUT, b"pvf1".to_vec(), PreparePriority::Normal, - ExecutePriority::Normal, + PvfExecPriority::Backing, result_tx, ) .await @@ -1270,7 +1270,7 @@ pub(crate) mod tests { TEST_EXECUTION_TIMEOUT, b"pvf1".to_vec(), PreparePriority::Critical, - ExecutePriority::Normal, + PvfExecPriority::Backing, result_tx, ) .await @@ -1282,7 +1282,7 @@ pub(crate) mod tests { TEST_EXECUTION_TIMEOUT, b"pvf2".to_vec(), PreparePriority::Normal, - ExecutePriority::Normal, + PvfExecPriority::Backing, result_tx, ) .await @@ -1424,7 +1424,7 @@ pub(crate) mod tests { TEST_EXECUTION_TIMEOUT, b"pvf2".to_vec(), PreparePriority::Critical, - ExecutePriority::Normal, + PvfExecPriority::Backing, result_tx, ) .await @@ -1472,7 +1472,7 @@ pub(crate) mod tests { TEST_EXECUTION_TIMEOUT, b"pvf2".to_vec(), PreparePriority::Critical, - ExecutePriority::Normal, + PvfExecPriority::Backing, result_tx, ) .await @@ -1575,7 +1575,7 @@ pub(crate) mod tests { TEST_EXECUTION_TIMEOUT, b"pvf".to_vec(), PreparePriority::Critical, - ExecutePriority::Normal, + PvfExecPriority::Backing, result_tx, ) .await @@ -1606,7 +1606,7 @@ pub(crate) mod tests { TEST_EXECUTION_TIMEOUT, b"pvf".to_vec(), PreparePriority::Critical, - ExecutePriority::Normal, + PvfExecPriority::Backing, result_tx_2, ) .await @@ -1629,7 +1629,7 @@ pub(crate) mod tests { TEST_EXECUTION_TIMEOUT, b"pvf".to_vec(), PreparePriority::Critical, - ExecutePriority::Normal, + PvfExecPriority::Backing, result_tx_3, ) .await @@ -1680,7 +1680,7 @@ pub(crate) mod tests { TEST_EXECUTION_TIMEOUT, b"pvf".to_vec(), PreparePriority::Critical, - ExecutePriority::Normal, + PvfExecPriority::Backing, result_tx, ) .await @@ -1711,7 +1711,7 @@ pub(crate) mod tests { TEST_EXECUTION_TIMEOUT, b"pvf".to_vec(), PreparePriority::Critical, - ExecutePriority::Normal, + PvfExecPriority::Backing, result_tx_2, ) .await @@ -1734,7 +1734,7 @@ pub(crate) mod tests { TEST_EXECUTION_TIMEOUT, b"pvf".to_vec(), PreparePriority::Critical, - ExecutePriority::Normal, + PvfExecPriority::Backing, result_tx_3, ) .await @@ -1801,7 +1801,7 @@ pub(crate) mod tests { TEST_EXECUTION_TIMEOUT, b"pvf1".to_vec(), PreparePriority::Normal, - ExecutePriority::Normal, + PvfExecPriority::Backing, result_tx, ) .await diff --git a/polkadot/node/core/pvf/src/lib.rs b/polkadot/node/core/pvf/src/lib.rs index 64a1e1d22fc3a..1be9a9e12fd18 100644 --- a/polkadot/node/core/pvf/src/lib.rs +++ b/polkadot/node/core/pvf/src/lib.rs @@ -110,7 +110,7 @@ pub use host::{ PREPARE_BINARY_NAME, }; pub use metrics::Metrics; -pub use priority::{ExecutePriority, PreparePriority}; +pub use priority::PreparePriority; pub use worker_interface::{framed_recv, framed_send, JOB_TIMEOUT_WALL_CLOCK_FACTOR}; // Re-export some common types. diff --git a/polkadot/node/core/pvf/src/priority.rs b/polkadot/node/core/pvf/src/priority.rs index 5190673f9479f..82de559ed32ee 100644 --- a/polkadot/node/core/pvf/src/priority.rs +++ b/polkadot/node/core/pvf/src/priority.rs @@ -14,8 +14,6 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use polkadot_node_subsystem::messages::PvfExecutionPriority; - /// A priority assigned to preparation of a PVF. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] pub enum PreparePriority { @@ -37,56 +35,3 @@ impl PreparePriority { self == PreparePriority::Critical } } - -impl From for PreparePriority { - fn from(priority: PvfExecutionPriority) -> Self { - match priority { - PvfExecutionPriority::Backing => PreparePriority::Normal, - PvfExecutionPriority::Approval => PreparePriority::Critical, - PvfExecutionPriority::Dispute => PreparePriority::Critical, - } - } -} - -/// A priority assigned to execution of a PVF. -#[derive(Debug, Hash, PartialEq, Eq, PartialOrd, Ord, Clone, Copy)] -pub enum ExecutePriority { - /// Low - Low, - /// Normal - Normal, - /// Critical - Critical, -} - -impl From for ExecutePriority { - fn from(priority: PvfExecutionPriority) -> Self { - match priority { - PvfExecutionPriority::Backing => ExecutePriority::Low, - PvfExecutionPriority::Approval => ExecutePriority::Normal, - PvfExecutionPriority::Dispute => ExecutePriority::Critical, - } - } -} - -impl ExecutePriority { - /// Returns an iterator over the variants of `ExecutePriority` in order from `Low` to - /// `Critical`. - pub fn iter() -> impl Iterator { - [Self::Low, Self::Normal, Self::Critical].iter().copied() - } - - /// Returns the next lower priority level, or `None` if `self` is `Low`. - pub fn lower(&self) -> Option { - match self { - Self::Critical => Some(Self::Normal), - Self::Normal => Some(Self::Low), - Self::Low => None, - } - } - - /// Returns `true` if `self` is `Normal` - pub fn is_normal(&self) -> bool { - *self == Self::Normal - } -} diff --git a/polkadot/node/overseer/examples/minimal-example.rs b/polkadot/node/overseer/examples/minimal-example.rs index c22ca27aa585e..4f8e05d285ce0 100644 --- a/polkadot/node/overseer/examples/minimal-example.rs +++ b/polkadot/node/overseer/examples/minimal-example.rs @@ -24,7 +24,7 @@ use orchestra::async_trait; use std::time::Duration; use polkadot_node_primitives::{BlockData, PoV}; -use polkadot_node_subsystem_types::messages::{CandidateValidationMessage, PvfExecutionPriority}; +use polkadot_node_subsystem_types::messages::{CandidateValidationMessage, PvfExecPriority}; use polkadot_overseer::{ self as overseer, dummy::dummy_overseer_builder, @@ -77,7 +77,7 @@ impl Subsystem1 { candidate_receipt, pov: PoV { block_data: BlockData(Vec::new()) }.into(), executor_params: Default::default(), - exec_kind: PvfExecutionPriority::Backing, + exec_kind: PvfExecPriority::Backing, response_sender: tx, }; ctx.send_message(msg).await; diff --git a/polkadot/node/overseer/src/tests.rs b/polkadot/node/overseer/src/tests.rs index d60bd53ba0564..a2c3ffc6a049d 100644 --- a/polkadot/node/overseer/src/tests.rs +++ b/polkadot/node/overseer/src/tests.rs @@ -25,7 +25,7 @@ use polkadot_node_primitives::{ }; use polkadot_node_subsystem_test_helpers::mock::{dummy_unpin_handle, new_leaf}; use polkadot_node_subsystem_types::messages::{ - NetworkBridgeEvent, PvfExecutionPriority, ReportPeerMessage, RuntimeApiRequest, + NetworkBridgeEvent, PvfExecPriority, ReportPeerMessage, RuntimeApiRequest, }; use polkadot_primitives::{ CandidateHash, CandidateReceipt, CollatorPair, Id as ParaId, InvalidDisputeStatementKind, @@ -108,7 +108,7 @@ where candidate_receipt, pov: PoV { block_data: BlockData(Vec::new()) }.into(), executor_params: Default::default(), - exec_kind: PvfExecutionPriority::Backing, + exec_kind: PvfExecPriority::Backing, response_sender: tx, }) .await; @@ -806,7 +806,7 @@ fn test_candidate_validation_msg() -> CandidateValidationMessage { candidate_receipt, pov, executor_params: Default::default(), - exec_kind: PvfExecutionPriority::Backing, + exec_kind: PvfExecPriority::Backing, response_sender, } } diff --git a/polkadot/node/subsystem-types/src/messages.rs b/polkadot/node/subsystem-types/src/messages.rs index 7a578cec17ea0..baea29a6a8609 100644 --- a/polkadot/node/subsystem-types/src/messages.rs +++ b/polkadot/node/subsystem-types/src/messages.rs @@ -160,7 +160,7 @@ pub enum CandidateValidationMessage { /// Session's executor parameters executor_params: ExecutorParams, /// Execution kind, used for timeouts and retries (backing/approvals) - exec_kind: PvfExecutionPriority, + exec_kind: PvfExecPriority, /// The sending side of the response channel response_sender: oneshot::Sender>, }, @@ -185,7 +185,7 @@ pub enum CandidateValidationMessage { /// Session's executor parameters executor_params: ExecutorParams, /// Execution kind, used for timeouts and retries (backing/approvals) - exec_kind: PvfExecutionPriority, + exec_kind: PvfExecPriority, /// The sending side of the response channel response_sender: oneshot::Sender>, }, @@ -206,22 +206,44 @@ pub enum CandidateValidationMessage { /// Extends primitives::PvfExecKind to have a separate value for duspute requests /// which is important for prioritization. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum PvfExecutionPriority { - /// For backing requests. - Backing, - /// For approval requests - Approval, +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum PvfExecPriority { /// For dispute requests Dispute, + /// For approval requests + Approval, + /// For backing requests from system parachains. + BackingSystem, + /// For backing requests. + Backing, } -impl From for PvfExecKind { - fn from(exec: PvfExecutionPriority) -> Self { +impl From for PvfExecKind { + fn from(exec: PvfExecPriority) -> Self { match exec { - PvfExecutionPriority::Backing => PvfExecKind::Backing, - PvfExecutionPriority::Approval => PvfExecKind::Approval, - PvfExecutionPriority::Dispute => PvfExecKind::Approval, + PvfExecPriority::Dispute => PvfExecKind::Approval, + PvfExecPriority::Approval => PvfExecKind::Approval, + PvfExecPriority::BackingSystem => PvfExecKind::Backing, + PvfExecPriority::Backing => PvfExecKind::Backing, + } + } +} + +impl PvfExecPriority { + /// Returns an iterator over the variants of `PvfExecPriorityKind` in order + pub fn iter() -> impl Iterator { + [Self::Dispute, Self::Approval, Self::BackingSystem, Self::Backing] + .iter() + .copied() + } + + /// Returns the next lower priority level, or `None` if no more levels. + pub fn lower(&self) -> Option { + match self { + Self::Dispute => Some(Self::Approval), + Self::Approval => Some(Self::BackingSystem), + Self::BackingSystem => Some(Self::Backing), + Self::Backing => None, } } } From 7442c99d083c3e5a0b634547732729f1f3294106 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Wed, 10 Jul 2024 16:08:51 +0200 Subject: [PATCH 14/48] Fix --- polkadot/node/core/candidate-validation/src/lib.rs | 8 ++++---- polkadot/node/core/candidate-validation/src/tests.rs | 4 ++-- polkadot/node/core/pvf/tests/it/main.rs | 3 ++- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/polkadot/node/core/candidate-validation/src/lib.rs b/polkadot/node/core/candidate-validation/src/lib.rs index 1d9b641ca8798..5e2a6cd42b56a 100644 --- a/polkadot/node/core/candidate-validation/src/lib.rs +++ b/polkadot/node/core/candidate-validation/src/lib.rs @@ -660,7 +660,7 @@ async fn validate_candidate_exhaustive( let result = match exec_kind { // Retry is disabled to reduce the chance of nondeterministic blocks getting backed and // honest backers getting slashed. - PvfExecPriority::Backing => { + PvfExecPriority::Backing | PvfExecPriority::BackingSystem => { let prep_timeout = pvf_prep_timeout(&executor_params, PvfPrepKind::Prepare); let exec_timeout = pvf_exec_timeout(&executor_params, exec_kind.into()); let pvf = PvfPrepData::from_code( @@ -775,7 +775,7 @@ trait ValidationBackend { // The priority for the preparation job. prepare_priority: polkadot_node_core_pvf::PreparePriority, // The priority for the preparation job. - execute_priority: polkadot_node_core_pvf::ExecutePriority, + execute_priority: PvfExecPriority, ) -> Result; /// Tries executing a PVF. Will retry once if an error is encountered that may have @@ -796,7 +796,7 @@ trait ValidationBackend { // The priority for the preparation job. prepare_priority: polkadot_node_core_pvf::PreparePriority, // The priority for the preparation job. - execute_priority: polkadot_node_core_pvf::ExecutePriority, + execute_priority: PvfExecPriority, ) -> Result { let prep_timeout = pvf_prep_timeout(&executor_params, PvfPrepKind::Prepare); // Construct the PVF a single time, since it is an expensive operation. Cloning it is cheap. @@ -922,7 +922,7 @@ impl ValidationBackend for ValidationHost { // The priority for the preparation job. prepare_priority: polkadot_node_core_pvf::PreparePriority, // The priority for the preparation job. - execute_priority: polkadot_node_core_pvf::ExecutePriority, + execute_priority: PvfExecPriority, ) -> Result { let (tx, rx) = oneshot::channel(); if let Err(err) = self diff --git a/polkadot/node/core/candidate-validation/src/tests.rs b/polkadot/node/core/candidate-validation/src/tests.rs index 887c37ac6b878..b95c25d0b9363 100644 --- a/polkadot/node/core/candidate-validation/src/tests.rs +++ b/polkadot/node/core/candidate-validation/src/tests.rs @@ -378,7 +378,7 @@ impl ValidationBackend for MockValidateCandidateBackend { _timeout: Duration, _encoded_params: Vec, _prepare_priority: polkadot_node_core_pvf::PreparePriority, - _execute_priority: polkadot_node_core_pvf::ExecutePriority, + _execute_priority: PvfExecPriority, ) -> Result { // This is expected to panic if called more times than expected, indicating an error in the // test. @@ -1065,7 +1065,7 @@ impl ValidationBackend for MockPreCheckBackend { _timeout: Duration, _encoded_params: Vec, _prepare_priority: polkadot_node_core_pvf::PreparePriority, - _execute_priority: polkadot_node_core_pvf::ExecutePriority, + _execute_priority: PvfExecPriority, ) -> Result { unreachable!() } diff --git a/polkadot/node/core/pvf/tests/it/main.rs b/polkadot/node/core/pvf/tests/it/main.rs index b14e583c768c9..f7a5850d93261 100644 --- a/polkadot/node/core/pvf/tests/it/main.rs +++ b/polkadot/node/core/pvf/tests/it/main.rs @@ -25,6 +25,7 @@ use polkadot_node_core_pvf::{ PossiblyInvalidError, PrepareError, PrepareJobKind, PvfPrepData, ValidationError, ValidationHost, JOB_TIMEOUT_WALL_CLOCK_FACTOR, }; +use polkadot_node_subsystem::messages::PvfExecPriority; use polkadot_parachain_primitives::primitives::{BlockData, ValidationParams, ValidationResult}; use polkadot_primitives::{ExecutorParam, ExecutorParams, PvfExecKind, PvfPrepKind}; @@ -124,7 +125,7 @@ impl TestHost { TEST_EXECUTION_TIMEOUT, params.encode(), polkadot_node_core_pvf::PreparePriority::Normal, - polkadot_node_core_pvf::ExecutePriority::Normal, + PvfExecPriority::Backing, result_tx, ) .await From cb1353371b0df387d7ad11afd5ff816b14ed81b0 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Wed, 10 Jul 2024 17:39:45 +0200 Subject: [PATCH 15/48] Fix --- polkadot/node/core/candidate-validation/src/lib.rs | 10 ++-------- polkadot/node/core/pvf/src/execute/queue.rs | 7 ++++--- polkadot/node/core/pvf/src/priority.rs | 13 +++++++++++++ 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/polkadot/node/core/candidate-validation/src/lib.rs b/polkadot/node/core/candidate-validation/src/lib.rs index 5e2a6cd42b56a..5b870c02e37da 100644 --- a/polkadot/node/core/candidate-validation/src/lib.rs +++ b/polkadot/node/core/candidate-validation/src/lib.rs @@ -671,13 +671,7 @@ async fn validate_candidate_exhaustive( ); validation_backend - .validate_candidate( - pvf, - exec_timeout, - params.encode(), - exec_kind.into(), - exec_kind.into(), - ) + .validate_candidate(pvf, exec_timeout, params.encode(), exec_kind.into(), exec_kind) .await }, PvfExecPriority::Approval | PvfExecPriority::Dispute => @@ -689,7 +683,7 @@ async fn validate_candidate_exhaustive( executor_params, PVF_APPROVAL_EXECUTION_RETRY_DELAY, exec_kind.into(), - exec_kind.into(), + exec_kind, ) .await, }; diff --git a/polkadot/node/core/pvf/src/execute/queue.rs b/polkadot/node/core/pvf/src/execute/queue.rs index 464e015c5220d..8eb04daa44d03 100644 --- a/polkadot/node/core/pvf/src/execute/queue.rs +++ b/polkadot/node/core/pvf/src/execute/queue.rs @@ -625,10 +625,11 @@ struct Unscheduled { unscheduled: HashMap>, } -fn is_probable(probability: u8) -> bool { +// Return a bool with a probability of numerator/100 of being true, +// which means the numerator can't be more than 100 +fn is_probable(numerator: u32) -> bool { let mut rng = rand::thread_rng(); - let n: u8 = rng.gen_range(0..100); - n <= probability + rng.gen_ratio(numerator, 100) } impl Unscheduled { diff --git a/polkadot/node/core/pvf/src/priority.rs b/polkadot/node/core/pvf/src/priority.rs index 82de559ed32ee..878a673ec312d 100644 --- a/polkadot/node/core/pvf/src/priority.rs +++ b/polkadot/node/core/pvf/src/priority.rs @@ -14,6 +14,8 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . +use polkadot_node_subsystem::messages::PvfExecPriority; + /// A priority assigned to preparation of a PVF. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] pub enum PreparePriority { @@ -35,3 +37,14 @@ impl PreparePriority { self == PreparePriority::Critical } } + +impl From for PreparePriority { + fn from(priority: PvfExecPriority) -> Self { + match priority { + PvfExecPriority::Dispute => PreparePriority::Critical, + PvfExecPriority::Approval => PreparePriority::Critical, + PvfExecPriority::BackingSystem => PreparePriority::Normal, + PvfExecPriority::Backing => PreparePriority::Normal, + } + } +} From 4ec2e5e6863b3779596d3da699458038e7b3f326 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 11 Jul 2024 13:07:00 +0200 Subject: [PATCH 16/48] Fix --- polkadot/node/core/pvf/src/execute/queue.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polkadot/node/core/pvf/src/execute/queue.rs b/polkadot/node/core/pvf/src/execute/queue.rs index 8eb04daa44d03..d8f13188bfcd2 100644 --- a/polkadot/node/core/pvf/src/execute/queue.rs +++ b/polkadot/node/core/pvf/src/execute/queue.rs @@ -646,7 +646,7 @@ impl Unscheduled { fn select_next_priority(&self, is_probable: F) -> PvfExecPriority where - F: Fn(u8) -> bool, + F: Fn(u32) -> bool, { let mut priority = self.next_priority(is_probable); @@ -660,7 +660,7 @@ impl Unscheduled { fn next_priority(&self, is_probable: F) -> PvfExecPriority where - F: Fn(u8) -> bool, + F: Fn(u32) -> bool, { use PvfExecPriority::*; From b70c247003bedec5f67d9d0f40553832b505dd2d Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Wed, 17 Jul 2024 18:16:25 +0200 Subject: [PATCH 17/48] Assign system backing --- Cargo.lock | 1 + polkadot/node/core/backing/Cargo.toml | 1 + polkadot/node/core/backing/src/lib.rs | 8 +++++++- polkadot/node/core/backing/src/tests/mod.rs | 20 +++++++++---------- .../src/tests/prospective_parachains.rs | 2 +- 5 files changed, 20 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index a8b08d280158f..c551f270ecc0e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13051,6 +13051,7 @@ dependencies = [ "polkadot-node-subsystem", "polkadot-node-subsystem-test-helpers", "polkadot-node-subsystem-util", + "polkadot-parachain-primitives", "polkadot-primitives", "polkadot-primitives-test-helpers", "polkadot-statement-table", diff --git a/polkadot/node/core/backing/Cargo.toml b/polkadot/node/core/backing/Cargo.toml index ffd6de0768894..fe6e367149b2e 100644 --- a/polkadot/node/core/backing/Cargo.toml +++ b/polkadot/node/core/backing/Cargo.toml @@ -14,6 +14,7 @@ futures = "0.3.30" sp-keystore = { path = "../../../../substrate/primitives/keystore" } polkadot-primitives = { path = "../../../primitives" } polkadot-node-primitives = { path = "../../primitives" } +polkadot-parachain-primitives = { path = "../../../parachain" } polkadot-node-subsystem = { path = "../../subsystem" } polkadot-node-subsystem-util = { path = "../../subsystem-util" } polkadot-erasure-coding = { path = "../../../erasure-coding" } diff --git a/polkadot/node/core/backing/src/lib.rs b/polkadot/node/core/backing/src/lib.rs index 46cf9696161ab..e16800cdabd25 100644 --- a/polkadot/node/core/backing/src/lib.rs +++ b/polkadot/node/core/backing/src/lib.rs @@ -105,6 +105,7 @@ use polkadot_node_subsystem_util::{ }, Validator, }; +use polkadot_parachain_primitives::primitives::IsSystem; use polkadot_primitives::{ node_features::FeatureIndex, BackedCandidate, CandidateCommitments, CandidateHash, CandidateReceipt, CommittedCandidateReceipt, CoreIndex, CoreState, ExecutorParams, GroupIndex, @@ -624,6 +625,7 @@ async fn request_candidate_validation( executor_params: ExecutorParams, ) -> Result { let (tx, rx) = oneshot::channel(); + let is_system = candidate_receipt.descriptor.para_id.is_system(); sender .send_message(CandidateValidationMessage::ValidateFromExhaustive { @@ -632,7 +634,11 @@ async fn request_candidate_validation( candidate_receipt, pov, executor_params, - exec_kind: PvfExecPriority::Backing, + exec_kind: if is_system { + PvfExecPriority::BackingSystem + } else { + PvfExecPriority::Backing + }, response_sender: tx, }) .await; diff --git a/polkadot/node/core/backing/src/tests/mod.rs b/polkadot/node/core/backing/src/tests/mod.rs index 0378357649f5f..d3c6dca97b0ae 100644 --- a/polkadot/node/core/backing/src/tests/mod.rs +++ b/polkadot/node/core/backing/src/tests/mod.rs @@ -405,7 +405,7 @@ async fn assert_validate_from_exhaustive( ) if validation_data == *assert_pvd && validation_code == *assert_validation_code && *pov == *assert_pov && &candidate_receipt.descriptor == assert_candidate.descriptor() && - exec_kind == PvfExecPriority::Backing && + exec_kind == PvfExecPriority::BackingSystem && candidate_receipt.commitments_hash == assert_candidate.commitments.hash() => { response_sender.send(Ok(ValidationResult::Valid( @@ -622,7 +622,7 @@ fn backing_works(#[case] elastic_scaling_mvp: bool) { ) if validation_data == pvd_ab && validation_code == validation_code_ab && *pov == pov_ab && &candidate_receipt.descriptor == candidate_a.descriptor() && - exec_kind == PvfExecPriority::Backing && + exec_kind == PvfExecPriority::BackingSystem && candidate_receipt.commitments_hash == candidate_a_commitments_hash => { response_sender.send(Ok( @@ -1250,7 +1250,7 @@ fn backing_works_while_validation_ongoing() { ) if validation_data == pvd_abc && validation_code == validation_code_abc && *pov == pov_abc && &candidate_receipt.descriptor == candidate_a.descriptor() && - exec_kind == PvfExecPriority::Backing && + exec_kind == PvfExecPriority::BackingSystem && candidate_a_commitments_hash == candidate_receipt.commitments_hash => { // we never validate the candidate. our local node @@ -1417,7 +1417,7 @@ fn backing_misbehavior_works() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_a && &candidate_receipt.descriptor == candidate_a.descriptor() && - exec_kind == PvfExecPriority::Backing && + exec_kind == PvfExecPriority::BackingSystem && candidate_a_commitments_hash == candidate_receipt.commitments_hash => { response_sender.send(Ok( @@ -1584,7 +1584,7 @@ fn backing_dont_second_invalid() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_block_a && &candidate_receipt.descriptor == candidate_a.descriptor() && - exec_kind == PvfExecPriority::Backing && + exec_kind == PvfExecPriority::BackingSystem && candidate_a.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Ok(ValidationResult::Invalid(InvalidCandidate::BadReturn))).unwrap(); @@ -1624,7 +1624,7 @@ fn backing_dont_second_invalid() { ) if validation_data == pvd_b && validation_code == validation_code_b && *pov == pov_block_b && &candidate_receipt.descriptor == candidate_b.descriptor() && - exec_kind == PvfExecPriority::Backing && + exec_kind == PvfExecPriority::BackingSystem && candidate_b.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Ok( @@ -1751,7 +1751,7 @@ fn backing_second_after_first_fails_works() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_a && &candidate_receipt.descriptor == candidate.descriptor() && - exec_kind == PvfExecPriority::Backing && + exec_kind == PvfExecPriority::BackingSystem && candidate.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Ok(ValidationResult::Invalid(InvalidCandidate::BadReturn))).unwrap(); @@ -1895,7 +1895,7 @@ fn backing_works_after_failed_validation() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_a && &candidate_receipt.descriptor == candidate.descriptor() && - exec_kind == PvfExecPriority::Backing && + exec_kind == PvfExecPriority::BackingSystem && candidate.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Err(ValidationFailed("Internal test error".into()))).unwrap(); @@ -2174,7 +2174,7 @@ fn retry_works() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_a && &candidate_receipt.descriptor == candidate.descriptor() && - exec_kind == PvfExecPriority::Backing && + exec_kind == PvfExecPriority::BackingSystem && candidate.commitments.hash() == candidate_receipt.commitments_hash ); virtual_overseer @@ -2716,7 +2716,7 @@ fn validator_ignores_statements_from_disabled_validators() { ) if validation_data == pvd && validation_code == expected_validation_code && *pov == expected_pov && &candidate_receipt.descriptor == candidate.descriptor() && - exec_kind == PvfExecPriority::Backing && + exec_kind == PvfExecPriority::BackingSystem && candidate_commitments_hash == candidate_receipt.commitments_hash => { response_sender.send(Ok( diff --git a/polkadot/node/core/backing/src/tests/prospective_parachains.rs b/polkadot/node/core/backing/src/tests/prospective_parachains.rs index f3aff6ba96be6..c36d03286f8c3 100644 --- a/polkadot/node/core/backing/src/tests/prospective_parachains.rs +++ b/polkadot/node/core/backing/src/tests/prospective_parachains.rs @@ -256,7 +256,7 @@ async fn assert_validate_seconded_candidate( &validation_code == assert_validation_code && &*pov == assert_pov && &candidate_receipt.descriptor == candidate.descriptor() && - exec_kind == PvfExecPriority::Backing && + exec_kind == PvfExecPriority::BackingSystem && candidate.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Ok(ValidationResult::Valid( From 2dfbffc3aa8ea45f8e5e19baba0da3d6855165f7 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 18 Jul 2024 19:57:50 +0200 Subject: [PATCH 18/48] Replace probability with counter --- polkadot/node/core/pvf/src/execute/queue.rs | 165 +++++++++++------- polkadot/node/subsystem-types/src/messages.rs | 2 +- 2 files changed, 106 insertions(+), 61 deletions(-) diff --git a/polkadot/node/core/pvf/src/execute/queue.rs b/polkadot/node/core/pvf/src/execute/queue.rs index d8f13188bfcd2..291936f88f9a1 100644 --- a/polkadot/node/core/pvf/src/execute/queue.rs +++ b/polkadot/node/core/pvf/src/execute/queue.rs @@ -36,7 +36,6 @@ use polkadot_node_core_pvf_common::{ }; use polkadot_node_subsystem::messages::PvfExecPriority; use polkadot_primitives::{ExecutorParams, ExecutorParamsHash}; -use rand::Rng; use slotmap::HopSlotMap; use std::{ collections::{HashMap, VecDeque}, @@ -228,7 +227,7 @@ impl Queue { // New jobs are always pushed to the tail of the queue; the one at its head is always // the eldest one. - let priority = self.unscheduled.select_next_priority(is_probable); + let priority = self.unscheduled.select_next_priority(); let Some(queue) = self.unscheduled.get_mut(priority) else { return }; let eldest = if let Some(eldest) = queue.get(0) { eldest } else { return }; @@ -280,6 +279,7 @@ impl Queue { } else { spawn_extra_worker(self, job); } + self.unscheduled.log(priority); } } @@ -623,64 +623,56 @@ pub fn start( struct Unscheduled { unscheduled: HashMap>, -} - -// Return a bool with a probability of numerator/100 of being true, -// which means the numerator can't be more than 100 -fn is_probable(numerator: u32) -> bool { - let mut rng = rand::thread_rng(); - rng.gen_ratio(numerator, 100) + counter: HashMap, } impl Unscheduled { // A threshold reaching which we prioritize approval jobs const REGULAR_JOBS_THRESHOLD: usize = 12; + const MAX_COUNT: usize = 1000; + const FULFILLED_THRESHOLDS: &'static [(PvfExecPriority, usize)] = &[ + (PvfExecPriority::Dispute, 70), + (PvfExecPriority::Approval, 87), + (PvfExecPriority::BackingSystem, 75), + ]; fn new() -> Self { Self { unscheduled: PvfExecPriority::iter() .map(|priority| (priority, VecDeque::new())) .collect(), + counter: PvfExecPriority::iter().map(|priority| (priority, 0)).collect(), } } - fn select_next_priority(&self, is_probable: F) -> PvfExecPriority - where - F: Fn(u32) -> bool, - { - let mut priority = self.next_priority(is_probable); - - while !self.has_pending(priority) { - let Some(p) = priority.lower() else { break }; - priority = p; - } - - priority - } - - fn next_priority(&self, is_probable: F) -> PvfExecPriority - where - F: Fn(u32) -> bool, - { + fn select_next_priority(&self) -> PvfExecPriority { use PvfExecPriority::*; - if self.has_pending(Dispute) && is_probable(70) { - return Dispute - } - - if !self.has_pending(Backing) && !self.has_pending(BackingSystem) { - return Approval - } + for priority in PvfExecPriority::iter() { + if !self.has_pending(priority) { + continue; + } - if self.regular_jobs_count() > Self::REGULAR_JOBS_THRESHOLD && is_probable(87) { - return Approval - } + match priority { + Approval => { + if !self.has_pending(Backing) && !self.has_pending(BackingSystem) { + return priority + } - if is_probable(75) { - return BackingSystem + if self.regular_jobs_count() > Self::REGULAR_JOBS_THRESHOLD && + !self.fulfilled(priority) + { + return priority + } + }, + _ => + if !self.fulfilled(priority) { + return priority + }, + } } - return Backing + Backing } // Number of jobs in all queues excluding disputes @@ -702,8 +694,50 @@ impl Unscheduled { self.unscheduled.entry(priority).or_default().push_back(job); } - fn has_pending(&self, kind: PvfExecPriority) -> bool { - !self.unscheduled.get(&kind).unwrap_or(&VecDeque::new()).is_empty() + fn has_pending(&self, priority: PvfExecPriority) -> bool { + !self.unscheduled.get(&priority).unwrap_or(&VecDeque::new()).is_empty() + } + + fn fulfilled_threshold(priority: PvfExecPriority) -> Option { + Self::FULFILLED_THRESHOLDS.iter().find_map( + |&(p, value)| { + if p == priority { + Some(value) + } else { + None + } + }, + ) + } + + fn fulfilled(&self, priority: PvfExecPriority) -> bool { + let Some(threshold) = Self::fulfilled_threshold(priority) else { return false }; + let Some(count) = self.counter.get(&priority) else { return false }; + // Every time we iterate by lower level priorities + let total_count: usize = self + .counter + .iter() + .filter_map(|(p, c)| if *p >= priority { Some(c) } else { None }) + .sum(); + if total_count == 0 { + return false + } + + count * 100 / total_count >= threshold + } + + fn log(&mut self, priority: PvfExecPriority) { + let current = self.counter.entry(priority).or_default(); + + if *current >= Self::MAX_COUNT { + return self.reset_counter(); + } + + *current += 1; + } + + fn reset_counter(&mut self) { + self.counter = PvfExecPriority::iter().map(|kind| (kind, 0)).collect(); } } @@ -745,39 +779,45 @@ mod tests { // With disputes let mut unscheduled = Unscheduled::new(); - unscheduled.add(create_execution_job(), Approval); unscheduled.add(create_execution_job(), Backing); - unscheduled.add(create_execution_job(), BackingSystem); unscheduled.add(create_execution_job(), Dispute); - // Do disputes with probability 70% - assert_eq!(unscheduled.select_next_priority(|p| p == 70), Dispute); - assert_eq!(unscheduled.select_next_priority(|_| false), Backing); + // Do disputes until fulfilled + let threshold = Unscheduled::fulfilled_threshold(Dispute).unwrap(); + (1..=(100 - threshold)).for_each(|_| unscheduled.log(Backing)); + (1..=(threshold - 1)).for_each(|_| unscheduled.log(Dispute)); + assert_eq!(unscheduled.select_next_priority(), Dispute); + unscheduled.log(Dispute); + assert_eq!(unscheduled.select_next_priority(), Backing); // Without disputes and backing + unscheduled.reset_counter(); unscheduled.get_mut(Dispute).unwrap().clear(); - unscheduled.get_mut(BackingSystem).unwrap().clear(); unscheduled.get_mut(Backing).unwrap().clear(); + unscheduled.add(create_execution_job(), Approval); // Do approval - assert_eq!(unscheduled.select_next_priority(|_| false), Approval); + assert_eq!(unscheduled.select_next_priority(), Approval); // But if no approvals unscheduled.get_mut(Approval).unwrap().clear(); - // Fallback to Backing as the lowest value - assert_eq!(unscheduled.select_next_priority(|_| true), Backing); + // // Fallback to Backing as the lowest value + assert_eq!(unscheduled.select_next_priority(), Backing); // With long queue for _ in 0..Unscheduled::REGULAR_JOBS_THRESHOLD { unscheduled.add(create_execution_job(), Approval); } - unscheduled.add(create_execution_job(), BackingSystem); unscheduled.add(create_execution_job(), Backing); - // Do approval with probability 87% - assert_eq!(unscheduled.select_next_priority(|p| p == 87), Approval); - assert_eq!(unscheduled.select_next_priority(|_| false), Backing); + // Do approval until fulfilled + let threshold = Unscheduled::fulfilled_threshold(Approval).unwrap(); + (1..=(100 - threshold)).for_each(|_| unscheduled.log(Backing)); + (1..=(threshold - 1)).for_each(|_| unscheduled.log(Approval)); + assert_eq!(unscheduled.select_next_priority(), Approval); + unscheduled.log(Approval); + assert_eq!(unscheduled.select_next_priority(), Backing); // But if no approvals unscheduled.get_mut(Approval).unwrap().clear(); @@ -786,19 +826,24 @@ mod tests { } // Fallback to next not empty queue priority - assert_eq!(unscheduled.select_next_priority(|_| true), BackingSystem); + assert_eq!(unscheduled.select_next_priority(), BackingSystem); // Otherwise + unscheduled.reset_counter(); unscheduled.get_mut(Approval).unwrap().clear(); - // Do system parachains backing with probability 75% - assert_eq!(unscheduled.select_next_priority(|p| p == 75), BackingSystem); - assert_eq!(unscheduled.select_next_priority(|_| false), Backing); + // Do system parachains backing until fulfilled + let threshold = Unscheduled::fulfilled_threshold(BackingSystem).unwrap(); + (1..=(100 - threshold)).for_each(|_| unscheduled.log(Backing)); + (1..=(threshold - 1)).for_each(|_| unscheduled.log(BackingSystem)); + assert_eq!(unscheduled.select_next_priority(), BackingSystem); + unscheduled.log(BackingSystem); + assert_eq!(unscheduled.select_next_priority(), Backing); // But if no system parachains backing unscheduled.get_mut(BackingSystem).unwrap().clear(); // Fallback to next not empty queue priority - assert_eq!(unscheduled.select_next_priority(|_| true), Backing); + assert_eq!(unscheduled.select_next_priority(), Backing); } } diff --git a/polkadot/node/subsystem-types/src/messages.rs b/polkadot/node/subsystem-types/src/messages.rs index baea29a6a8609..0ec9a2ac511b6 100644 --- a/polkadot/node/subsystem-types/src/messages.rs +++ b/polkadot/node/subsystem-types/src/messages.rs @@ -206,7 +206,7 @@ pub enum CandidateValidationMessage { /// Extends primitives::PvfExecKind to have a separate value for duspute requests /// which is important for prioritization. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] pub enum PvfExecPriority { /// For dispute requests Dispute, From 98a4e64ffcf76387311e162ec628607c3e6228fd Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 19 Jul 2024 10:52:44 +0200 Subject: [PATCH 19/48] Update iteration --- Cargo.lock | 2 ++ polkadot/node/core/pvf/Cargo.toml | 1 + polkadot/node/core/pvf/src/execute/queue.rs | 1 + polkadot/node/subsystem-types/Cargo.toml | 1 + polkadot/node/subsystem-types/src/messages.rs | 24 +++---------------- 5 files changed, 8 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 80211e506985a..0295a534402bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -13404,6 +13404,7 @@ dependencies = [ "slotmap", "sp-core", "sp-maybe-compressed-blob", + "strum 0.26.2", "tempfile", "test-parachain-adder", "test-parachain-halt", @@ -13671,6 +13672,7 @@ dependencies = [ "sp-blockchain", "sp-consensus-babe", "sp-runtime", + "strum 0.26.2", "substrate-prometheus-endpoint", "thiserror", ] diff --git a/polkadot/node/core/pvf/Cargo.toml b/polkadot/node/core/pvf/Cargo.toml index 7444f7927f568..54e5b80505bf0 100644 --- a/polkadot/node/core/pvf/Cargo.toml +++ b/polkadot/node/core/pvf/Cargo.toml @@ -24,6 +24,7 @@ slotmap = { workspace = true } tempfile = { workspace = true } thiserror = { workspace = true } tokio = { features = ["fs", "process"], workspace = true, default-features = true } +strum = { features = ["derive"], workspace = true, default-features = true } codec = { features = [ "derive", diff --git a/polkadot/node/core/pvf/src/execute/queue.rs b/polkadot/node/core/pvf/src/execute/queue.rs index 291936f88f9a1..0e38a50f82160 100644 --- a/polkadot/node/core/pvf/src/execute/queue.rs +++ b/polkadot/node/core/pvf/src/execute/queue.rs @@ -43,6 +43,7 @@ use std::{ path::PathBuf, time::{Duration, Instant}, }; +use strum::IntoEnumIterator; /// The amount of time a job for which the queue does not have a compatible worker may wait in the /// queue. After that time passes, the queue will kill the first worker which becomes idle to diff --git a/polkadot/node/subsystem-types/Cargo.toml b/polkadot/node/subsystem-types/Cargo.toml index c8fc324699e17..62f334e98ef40 100644 --- a/polkadot/node/subsystem-types/Cargo.toml +++ b/polkadot/node/subsystem-types/Cargo.toml @@ -33,3 +33,4 @@ prometheus-endpoint = { workspace = true, default-features = true } thiserror = { workspace = true } async-trait = { workspace = true } bitvec = { features = ["alloc"], workspace = true } +strum = { features = ["derive"], workspace = true, default-features = true } diff --git a/polkadot/node/subsystem-types/src/messages.rs b/polkadot/node/subsystem-types/src/messages.rs index e6293401ee1e6..ccbfca90f0190 100644 --- a/polkadot/node/subsystem-types/src/messages.rs +++ b/polkadot/node/subsystem-types/src/messages.rs @@ -24,6 +24,7 @@ use futures::channel::oneshot; use sc_network::{Multiaddr, ReputationChange}; +use strum::EnumIter; use thiserror::Error; pub use sc_network::IfDisconnected; @@ -204,9 +205,9 @@ pub enum CandidateValidationMessage { }, } -/// Extends primitives::PvfExecKind to have a separate value for duspute requests +/// Extends primitives::PvfExecKind to have a separate value for dispute requests /// which is important for prioritization. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, EnumIter)] pub enum PvfExecPriority { /// For dispute requests Dispute, @@ -229,25 +230,6 @@ impl From for PvfExecKind { } } -impl PvfExecPriority { - /// Returns an iterator over the variants of `PvfExecPriorityKind` in order - pub fn iter() -> impl Iterator { - [Self::Dispute, Self::Approval, Self::BackingSystem, Self::Backing] - .iter() - .copied() - } - - /// Returns the next lower priority level, or `None` if no more levels. - pub fn lower(&self) -> Option { - match self { - Self::Dispute => Some(Self::Approval), - Self::Approval => Some(Self::BackingSystem), - Self::BackingSystem => Some(Self::Backing), - Self::Backing => None, - } - } -} - /// Messages received by the Collator Protocol subsystem. #[derive(Debug, derive_more::From)] pub enum CollatorProtocolMessage { From 5060a8144e8c45103ec5552e3440380d3bcfb863 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 19 Jul 2024 11:09:18 +0200 Subject: [PATCH 20/48] Update description --- polkadot/node/subsystem-types/src/messages.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polkadot/node/subsystem-types/src/messages.rs b/polkadot/node/subsystem-types/src/messages.rs index ccbfca90f0190..f1bc57578b438 100644 --- a/polkadot/node/subsystem-types/src/messages.rs +++ b/polkadot/node/subsystem-types/src/messages.rs @@ -205,8 +205,8 @@ pub enum CandidateValidationMessage { }, } -/// Extends primitives::PvfExecKind to have a separate value for dispute requests -/// which is important for prioritization. +/// Extends primitives::PvfExecKind, which is a runtime parameter we don't want to change, +/// to separate and prioritize execution jobs by request type. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, EnumIter)] pub enum PvfExecPriority { /// For dispute requests From 7e071f6ef214a82afb79978c38c8fd2af3270b9b Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 19 Jul 2024 12:03:55 +0200 Subject: [PATCH 21/48] Add logging --- polkadot/node/core/pvf/src/execute/queue.rs | 49 ++++++++++++++------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/polkadot/node/core/pvf/src/execute/queue.rs b/polkadot/node/core/pvf/src/execute/queue.rs index 0e38a50f82160..c487c5d57ba6c 100644 --- a/polkadot/node/core/pvf/src/execute/queue.rs +++ b/polkadot/node/core/pvf/src/execute/queue.rs @@ -649,30 +649,45 @@ impl Unscheduled { fn select_next_priority(&self) -> PvfExecPriority { use PvfExecPriority::*; + // Iterate over priorities starting with Dispute for priority in PvfExecPriority::iter() { if !self.has_pending(priority) { continue; } - match priority { - Approval => { - if !self.has_pending(Backing) && !self.has_pending(BackingSystem) { - return priority - } - - if self.regular_jobs_count() > Self::REGULAR_JOBS_THRESHOLD && - !self.fulfilled(priority) - { - return priority - } - }, - _ => - if !self.fulfilled(priority) { - return priority - }, + let is_fulfilled = self.is_fulfilled(priority); + if priority == Approval { + // To run jobs from Approval queue we should either + // - have no backing jobs + // - be overworked but not yet fulfilled + let has_backing_jobs = self.has_pending(Backing) || self.has_pending(BackingSystem); + let queue_size_without_disputes = self.regular_jobs_count(); + let queue_overworked = queue_size_without_disputes > Self::REGULAR_JOBS_THRESHOLD; + + if !has_backing_jobs || queue_overworked && !is_fulfilled { + gum::debug!( + target: LOG_TARGET, + ?has_backing_jobs, + ?queue_size_without_disputes, + ?queue_overworked, + "Chosen {} priority to run next", + priority + ); + return priority + } + } else if !is_fulfilled { + // Other priorities require only to be not fulfilled to run next + gum::debug!( + target: LOG_TARGET, + "Chosen {} priority to run next, its limit is not yet fulfilled", + priority + ); + + return priority } } + // The lowest priority as a fallback Backing } @@ -711,7 +726,7 @@ impl Unscheduled { ) } - fn fulfilled(&self, priority: PvfExecPriority) -> bool { + fn is_fulfilled(&self, priority: PvfExecPriority) -> bool { let Some(threshold) = Self::fulfilled_threshold(priority) else { return false }; let Some(count) = self.counter.get(&priority) else { return false }; // Every time we iterate by lower level priorities From 34b636328f26ed2c18daa57ad6a1d2d44b0ed9d6 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Mon, 22 Jul 2024 17:49:57 +0200 Subject: [PATCH 22/48] Update --- polkadot/node/core/backing/src/lib.rs | 2 +- polkadot/node/core/backing/src/tests/mod.rs | 20 +-- .../src/tests/prospective_parachains.rs | 2 +- .../node/core/candidate-validation/src/lib.rs | 2 +- polkadot/node/core/pvf/src/execute/queue.rs | 156 ++++-------------- polkadot/node/core/pvf/src/priority.rs | 2 +- polkadot/node/subsystem-types/src/messages.rs | 4 +- 7 files changed, 52 insertions(+), 136 deletions(-) diff --git a/polkadot/node/core/backing/src/lib.rs b/polkadot/node/core/backing/src/lib.rs index fb4242ad5e15e..6b99ef8ad50a7 100644 --- a/polkadot/node/core/backing/src/lib.rs +++ b/polkadot/node/core/backing/src/lib.rs @@ -637,7 +637,7 @@ async fn request_candidate_validation( pov, executor_params, exec_kind: if is_system { - PvfExecPriority::BackingSystem + PvfExecPriority::BackingSystemParas } else { PvfExecPriority::Backing }, diff --git a/polkadot/node/core/backing/src/tests/mod.rs b/polkadot/node/core/backing/src/tests/mod.rs index 754ee08078e54..6ed7b9c6d9795 100644 --- a/polkadot/node/core/backing/src/tests/mod.rs +++ b/polkadot/node/core/backing/src/tests/mod.rs @@ -434,7 +434,7 @@ async fn assert_validate_from_exhaustive( ) if validation_data == *assert_pvd && validation_code == *assert_validation_code && *pov == *assert_pov && &candidate_receipt.descriptor == assert_candidate.descriptor() && - exec_kind == PvfExecPriority::BackingSystem && + exec_kind == PvfExecPriority::BackingSystemParas && candidate_receipt.commitments_hash == assert_candidate.commitments.hash() => { response_sender.send(Ok(ValidationResult::Valid( @@ -651,7 +651,7 @@ fn backing_works(#[case] elastic_scaling_mvp: bool) { ) if validation_data == pvd_ab && validation_code == validation_code_ab && *pov == pov_ab && &candidate_receipt.descriptor == candidate_a.descriptor() && - exec_kind == PvfExecPriority::BackingSystem && + exec_kind == PvfExecPriority::BackingSystemParas && candidate_receipt.commitments_hash == candidate_a_commitments_hash => { response_sender.send(Ok( @@ -1287,7 +1287,7 @@ fn backing_works_while_validation_ongoing() { ) if validation_data == pvd_abc && validation_code == validation_code_abc && *pov == pov_abc && &candidate_receipt.descriptor == candidate_a.descriptor() && - exec_kind == PvfExecPriority::BackingSystem && + exec_kind == PvfExecPriority::BackingSystemParas && candidate_a_commitments_hash == candidate_receipt.commitments_hash => { // we never validate the candidate. our local node @@ -1454,7 +1454,7 @@ fn backing_misbehavior_works() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_a && &candidate_receipt.descriptor == candidate_a.descriptor() && - exec_kind == PvfExecPriority::BackingSystem && + exec_kind == PvfExecPriority::BackingSystemParas && candidate_a_commitments_hash == candidate_receipt.commitments_hash => { response_sender.send(Ok( @@ -1621,7 +1621,7 @@ fn backing_dont_second_invalid() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_block_a && &candidate_receipt.descriptor == candidate_a.descriptor() && - exec_kind == PvfExecPriority::BackingSystem && + exec_kind == PvfExecPriority::BackingSystemParas && candidate_a.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Ok(ValidationResult::Invalid(InvalidCandidate::BadReturn))).unwrap(); @@ -1661,7 +1661,7 @@ fn backing_dont_second_invalid() { ) if validation_data == pvd_b && validation_code == validation_code_b && *pov == pov_block_b && &candidate_receipt.descriptor == candidate_b.descriptor() && - exec_kind == PvfExecPriority::BackingSystem && + exec_kind == PvfExecPriority::BackingSystemParas && candidate_b.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Ok( @@ -1788,7 +1788,7 @@ fn backing_second_after_first_fails_works() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_a && &candidate_receipt.descriptor == candidate.descriptor() && - exec_kind == PvfExecPriority::BackingSystem && + exec_kind == PvfExecPriority::BackingSystemParas && candidate.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Ok(ValidationResult::Invalid(InvalidCandidate::BadReturn))).unwrap(); @@ -1932,7 +1932,7 @@ fn backing_works_after_failed_validation() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_a && &candidate_receipt.descriptor == candidate.descriptor() && - exec_kind == PvfExecPriority::BackingSystem && + exec_kind == PvfExecPriority::BackingSystemParas && candidate.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Err(ValidationFailed("Internal test error".into()))).unwrap(); @@ -2211,7 +2211,7 @@ fn retry_works() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_a && &candidate_receipt.descriptor == candidate.descriptor() && - exec_kind == PvfExecPriority::BackingSystem && + exec_kind == PvfExecPriority::BackingSystemParas && candidate.commitments.hash() == candidate_receipt.commitments_hash ); virtual_overseer @@ -2753,7 +2753,7 @@ fn validator_ignores_statements_from_disabled_validators() { ) if validation_data == pvd && validation_code == expected_validation_code && *pov == expected_pov && &candidate_receipt.descriptor == candidate.descriptor() && - exec_kind == PvfExecPriority::BackingSystem && + exec_kind == PvfExecPriority::BackingSystemParas && candidate_commitments_hash == candidate_receipt.commitments_hash => { response_sender.send(Ok( diff --git a/polkadot/node/core/backing/src/tests/prospective_parachains.rs b/polkadot/node/core/backing/src/tests/prospective_parachains.rs index 5e45010b4c0cf..3d7d4df0f1a64 100644 --- a/polkadot/node/core/backing/src/tests/prospective_parachains.rs +++ b/polkadot/node/core/backing/src/tests/prospective_parachains.rs @@ -276,7 +276,7 @@ async fn assert_validate_seconded_candidate( &validation_code == assert_validation_code && &*pov == assert_pov && &candidate_receipt.descriptor == candidate.descriptor() && - exec_kind == PvfExecPriority::BackingSystem && + exec_kind == PvfExecPriority::BackingSystemParas && candidate.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Ok(ValidationResult::Valid( diff --git a/polkadot/node/core/candidate-validation/src/lib.rs b/polkadot/node/core/candidate-validation/src/lib.rs index 5b870c02e37da..fda99f611147b 100644 --- a/polkadot/node/core/candidate-validation/src/lib.rs +++ b/polkadot/node/core/candidate-validation/src/lib.rs @@ -660,7 +660,7 @@ async fn validate_candidate_exhaustive( let result = match exec_kind { // Retry is disabled to reduce the chance of nondeterministic blocks getting backed and // honest backers getting slashed. - PvfExecPriority::Backing | PvfExecPriority::BackingSystem => { + PvfExecPriority::Backing | PvfExecPriority::BackingSystemParas => { let prep_timeout = pvf_prep_timeout(&executor_params, PvfPrepKind::Prepare); let exec_timeout = pvf_exec_timeout(&executor_params, exec_kind.into()); let pvf = PvfPrepData::from_code( diff --git a/polkadot/node/core/pvf/src/execute/queue.rs b/polkadot/node/core/pvf/src/execute/queue.rs index c487c5d57ba6c..03204a483e1d0 100644 --- a/polkadot/node/core/pvf/src/execute/queue.rs +++ b/polkadot/node/core/pvf/src/execute/queue.rs @@ -629,12 +629,12 @@ struct Unscheduled { impl Unscheduled { // A threshold reaching which we prioritize approval jobs - const REGULAR_JOBS_THRESHOLD: usize = 12; - const MAX_COUNT: usize = 1000; + const MAX_COUNT: usize = 12; const FULFILLED_THRESHOLDS: &'static [(PvfExecPriority, usize)] = &[ (PvfExecPriority::Dispute, 70), - (PvfExecPriority::Approval, 87), - (PvfExecPriority::BackingSystem, 75), + (PvfExecPriority::Approval, 80), + (PvfExecPriority::BackingSystemParas, 100), + (PvfExecPriority::Backing, 100), ]; fn new() -> Self { @@ -647,59 +647,10 @@ impl Unscheduled { } fn select_next_priority(&self) -> PvfExecPriority { - use PvfExecPriority::*; - - // Iterate over priorities starting with Dispute - for priority in PvfExecPriority::iter() { - if !self.has_pending(priority) { - continue; - } - - let is_fulfilled = self.is_fulfilled(priority); - if priority == Approval { - // To run jobs from Approval queue we should either - // - have no backing jobs - // - be overworked but not yet fulfilled - let has_backing_jobs = self.has_pending(Backing) || self.has_pending(BackingSystem); - let queue_size_without_disputes = self.regular_jobs_count(); - let queue_overworked = queue_size_without_disputes > Self::REGULAR_JOBS_THRESHOLD; - - if !has_backing_jobs || queue_overworked && !is_fulfilled { - gum::debug!( - target: LOG_TARGET, - ?has_backing_jobs, - ?queue_size_without_disputes, - ?queue_overworked, - "Chosen {} priority to run next", - priority - ); - return priority - } - } else if !is_fulfilled { - // Other priorities require only to be not fulfilled to run next - gum::debug!( - target: LOG_TARGET, - "Chosen {} priority to run next, its limit is not yet fulfilled", - priority - ); - - return priority - } - } - - // The lowest priority as a fallback - Backing - } - - // Number of jobs in all queues excluding disputes - fn regular_jobs_count(&self) -> usize { - let mut count = 0; - for (priority, queue) in self.unscheduled.iter() { - if *priority != PvfExecPriority::Dispute { - count += queue.len(); - } - } - count + PvfExecPriority::iter() + .filter(|priority| self.has_pending(priority) && !self.is_fulfilled(priority)) + .next() + .unwrap_or(PvfExecPriority::Backing) } fn get_mut(&mut self, priority: PvfExecPriority) -> Option<&mut VecDeque> { @@ -710,14 +661,14 @@ impl Unscheduled { self.unscheduled.entry(priority).or_default().push_back(job); } - fn has_pending(&self, priority: PvfExecPriority) -> bool { - !self.unscheduled.get(&priority).unwrap_or(&VecDeque::new()).is_empty() + fn has_pending(&self, priority: &PvfExecPriority) -> bool { + !self.unscheduled.get(priority).unwrap_or(&VecDeque::new()).is_empty() } - fn fulfilled_threshold(priority: PvfExecPriority) -> Option { + fn fulfilled_threshold(priority: &PvfExecPriority) -> Option { Self::FULFILLED_THRESHOLDS.iter().find_map( |&(p, value)| { - if p == priority { + if p == *priority { Some(value) } else { None @@ -726,14 +677,14 @@ impl Unscheduled { ) } - fn is_fulfilled(&self, priority: PvfExecPriority) -> bool { + fn is_fulfilled(&self, priority: &PvfExecPriority) -> bool { let Some(threshold) = Self::fulfilled_threshold(priority) else { return false }; let Some(count) = self.counter.get(&priority) else { return false }; // Every time we iterate by lower level priorities let total_count: usize = self .counter .iter() - .filter_map(|(p, c)| if *p >= priority { Some(c) } else { None }) + .filter_map(|(p, c)| if *p >= *priority { Some(c) } else { None }) .sum(); if total_count == 0 { return false @@ -743,13 +694,12 @@ impl Unscheduled { } fn log(&mut self, priority: PvfExecPriority) { - let current = self.counter.entry(priority).or_default(); + let current_count: &mut usize = self.counter.entry(priority).or_default(); + *current_count += 1; - if *current >= Self::MAX_COUNT { - return self.reset_counter(); + if self.counter.values().sum::() >= Self::MAX_COUNT { + self.reset_counter(); } - - *current += 1; } fn reset_counter(&mut self) { @@ -793,73 +743,39 @@ mod tests { fn test_unscheduled_select_next_priority() { use PvfExecPriority::*; - // With disputes let mut unscheduled = Unscheduled::new(); + + // With empty counter + assert_eq!(unscheduled.select_next_priority(), Backing); unscheduled.add(create_execution_job(), Backing); + assert_eq!(unscheduled.select_next_priority(), Backing); + unscheduled.add(create_execution_job(), BackingSystemParas); + assert_eq!(unscheduled.select_next_priority(), BackingSystemParas); + unscheduled.add(create_execution_job(), Approval); + assert_eq!(unscheduled.select_next_priority(), Approval); unscheduled.add(create_execution_job(), Dispute); - - // Do disputes until fulfilled - let threshold = Unscheduled::fulfilled_threshold(Dispute).unwrap(); - (1..=(100 - threshold)).for_each(|_| unscheduled.log(Backing)); - (1..=(threshold - 1)).for_each(|_| unscheduled.log(Dispute)); assert_eq!(unscheduled.select_next_priority(), Dispute); + + // Fulfill dispute jobs unscheduled.log(Dispute); - assert_eq!(unscheduled.select_next_priority(), Backing); + assert_eq!(unscheduled.select_next_priority(), Approval); - // Without disputes and backing + // Remove dispute jobs unscheduled.reset_counter(); unscheduled.get_mut(Dispute).unwrap().clear(); - unscheduled.get_mut(Backing).unwrap().clear(); - unscheduled.add(create_execution_job(), Approval); - - // Do approval assert_eq!(unscheduled.select_next_priority(), Approval); - // But if no approvals - unscheduled.get_mut(Approval).unwrap().clear(); - - // // Fallback to Backing as the lowest value - assert_eq!(unscheduled.select_next_priority(), Backing); - - // With long queue - for _ in 0..Unscheduled::REGULAR_JOBS_THRESHOLD { - unscheduled.add(create_execution_job(), Approval); - } - unscheduled.add(create_execution_job(), Backing); - - // Do approval until fulfilled - let threshold = Unscheduled::fulfilled_threshold(Approval).unwrap(); - (1..=(100 - threshold)).for_each(|_| unscheduled.log(Backing)); - (1..=(threshold - 1)).for_each(|_| unscheduled.log(Approval)); - assert_eq!(unscheduled.select_next_priority(), Approval); + // Fulfill approval jobs unscheduled.log(Approval); - assert_eq!(unscheduled.select_next_priority(), Backing); - - // But if no approvals - unscheduled.get_mut(Approval).unwrap().clear(); - for _ in 0..Unscheduled::REGULAR_JOBS_THRESHOLD { - unscheduled.add(create_execution_job(), BackingSystem); - } - - // Fallback to next not empty queue priority - assert_eq!(unscheduled.select_next_priority(), BackingSystem); + assert_eq!(unscheduled.select_next_priority(), BackingSystemParas); - // Otherwise + // Remove approval jobs unscheduled.reset_counter(); unscheduled.get_mut(Approval).unwrap().clear(); + assert_eq!(unscheduled.select_next_priority(), BackingSystemParas); - // Do system parachains backing until fulfilled - let threshold = Unscheduled::fulfilled_threshold(BackingSystem).unwrap(); - (1..=(100 - threshold)).for_each(|_| unscheduled.log(Backing)); - (1..=(threshold - 1)).for_each(|_| unscheduled.log(BackingSystem)); - assert_eq!(unscheduled.select_next_priority(), BackingSystem); - unscheduled.log(BackingSystem); - assert_eq!(unscheduled.select_next_priority(), Backing); - - // But if no system parachains backing - unscheduled.get_mut(BackingSystem).unwrap().clear(); - - // Fallback to next not empty queue priority + // Fulfill system parachains backing jobs + unscheduled.log(BackingSystemParas); assert_eq!(unscheduled.select_next_priority(), Backing); } } diff --git a/polkadot/node/core/pvf/src/priority.rs b/polkadot/node/core/pvf/src/priority.rs index 878a673ec312d..89ca3aaa34b83 100644 --- a/polkadot/node/core/pvf/src/priority.rs +++ b/polkadot/node/core/pvf/src/priority.rs @@ -43,7 +43,7 @@ impl From for PreparePriority { match priority { PvfExecPriority::Dispute => PreparePriority::Critical, PvfExecPriority::Approval => PreparePriority::Critical, - PvfExecPriority::BackingSystem => PreparePriority::Normal, + PvfExecPriority::BackingSystemParas => PreparePriority::Normal, PvfExecPriority::Backing => PreparePriority::Normal, } } diff --git a/polkadot/node/subsystem-types/src/messages.rs b/polkadot/node/subsystem-types/src/messages.rs index f1bc57578b438..359520060d1f4 100644 --- a/polkadot/node/subsystem-types/src/messages.rs +++ b/polkadot/node/subsystem-types/src/messages.rs @@ -214,7 +214,7 @@ pub enum PvfExecPriority { /// For approval requests Approval, /// For backing requests from system parachains. - BackingSystem, + BackingSystemParas, /// For backing requests. Backing, } @@ -224,7 +224,7 @@ impl From for PvfExecKind { match exec { PvfExecPriority::Dispute => PvfExecKind::Approval, PvfExecPriority::Approval => PvfExecKind::Approval, - PvfExecPriority::BackingSystem => PvfExecKind::Backing, + PvfExecPriority::BackingSystemParas => PvfExecKind::Backing, PvfExecPriority::Backing => PvfExecKind::Backing, } } From 531629de7592afa692bd40fc997b54def6377fc6 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Tue, 23 Jul 2024 11:53:46 +0200 Subject: [PATCH 23/48] Fix clippy warning --- polkadot/node/core/pvf/src/execute/queue.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/polkadot/node/core/pvf/src/execute/queue.rs b/polkadot/node/core/pvf/src/execute/queue.rs index 03204a483e1d0..12b642f10d5f4 100644 --- a/polkadot/node/core/pvf/src/execute/queue.rs +++ b/polkadot/node/core/pvf/src/execute/queue.rs @@ -648,8 +648,7 @@ impl Unscheduled { fn select_next_priority(&self) -> PvfExecPriority { PvfExecPriority::iter() - .filter(|priority| self.has_pending(priority) && !self.is_fulfilled(priority)) - .next() + .find(|priority| self.has_pending(priority) && !self.is_fulfilled(priority)) .unwrap_or(PvfExecPriority::Backing) } From 59b74bf91d2d7f5175e782a9492aeceab47440fa Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Tue, 23 Jul 2024 17:42:15 +0200 Subject: [PATCH 24/48] Update tests --- polkadot/node/core/candidate-validation/src/tests.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/polkadot/node/core/candidate-validation/src/tests.rs b/polkadot/node/core/candidate-validation/src/tests.rs index dac6f2c6fdba8..277d0f1174016 100644 --- a/polkadot/node/core/candidate-validation/src/tests.rs +++ b/polkadot/node/core/candidate-validation/src/tests.rs @@ -17,6 +17,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use super::*; +use crate::PvfExecPriority; use assert_matches::assert_matches; use futures::executor; use polkadot_node_core_pvf::PrepareError; @@ -1295,7 +1296,8 @@ impl ValidationBackend for MockHeadsUp { _pvf: PvfPrepData, _timeout: Duration, _encoded_params: Vec, - _prepare_priority: polkadot_node_core_pvf::Priority, + _prepare_priority: polkadot_node_core_pvf::PreparePriority, + _execute_priority: PvfExecPriority, ) -> Result { unreachable!() } From b261cc5327e3c6b995ac9d879d56ccb61b090a6c Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 25 Jul 2024 13:09:25 +0200 Subject: [PATCH 25/48] Rename PreparePriority back --- .../node/core/candidate-validation/src/lib.rs | 6 +-- .../core/candidate-validation/src/tests.rs | 6 +-- polkadot/node/core/pvf/src/host.rs | 43 +++++++++---------- polkadot/node/core/pvf/src/lib.rs | 2 +- polkadot/node/core/pvf/src/prepare/queue.rs | 40 ++++++++--------- polkadot/node/core/pvf/src/priority.rs | 16 +++---- polkadot/node/core/pvf/tests/it/main.rs | 2 +- 7 files changed, 56 insertions(+), 59 deletions(-) diff --git a/polkadot/node/core/candidate-validation/src/lib.rs b/polkadot/node/core/candidate-validation/src/lib.rs index e87d81fc4afd9..029be57fb5f6f 100644 --- a/polkadot/node/core/candidate-validation/src/lib.rs +++ b/polkadot/node/core/candidate-validation/src/lib.rs @@ -1018,7 +1018,7 @@ trait ValidationBackend { exec_timeout: Duration, encoded_params: Vec, // The priority for the preparation job. - prepare_priority: polkadot_node_core_pvf::PreparePriority, + prepare_priority: polkadot_node_core_pvf::Priority, // The priority for the preparation job. execute_priority: PvfExecPriority, ) -> Result; @@ -1039,7 +1039,7 @@ trait ValidationBackend { executor_params: ExecutorParams, retry_delay: Duration, // The priority for the preparation job. - prepare_priority: polkadot_node_core_pvf::PreparePriority, + prepare_priority: polkadot_node_core_pvf::Priority, // The priority for the preparation job. execute_priority: PvfExecPriority, ) -> Result { @@ -1167,7 +1167,7 @@ impl ValidationBackend for ValidationHost { exec_timeout: Duration, encoded_params: Vec, // The priority for the preparation job. - prepare_priority: polkadot_node_core_pvf::PreparePriority, + prepare_priority: polkadot_node_core_pvf::Priority, // The priority for the preparation job. execute_priority: PvfExecPriority, ) -> Result { diff --git a/polkadot/node/core/candidate-validation/src/tests.rs b/polkadot/node/core/candidate-validation/src/tests.rs index 277d0f1174016..f81099855b3e9 100644 --- a/polkadot/node/core/candidate-validation/src/tests.rs +++ b/polkadot/node/core/candidate-validation/src/tests.rs @@ -387,7 +387,7 @@ impl ValidationBackend for MockValidateCandidateBackend { _pvf: PvfPrepData, _timeout: Duration, _encoded_params: Vec, - _prepare_priority: polkadot_node_core_pvf::PreparePriority, + _prepare_priority: polkadot_node_core_pvf::Priority, _execute_priority: PvfExecPriority, ) -> Result { // This is expected to panic if called more times than expected, indicating an error in the @@ -1078,7 +1078,7 @@ impl ValidationBackend for MockPreCheckBackend { _pvf: PvfPrepData, _timeout: Duration, _encoded_params: Vec, - _prepare_priority: polkadot_node_core_pvf::PreparePriority, + _prepare_priority: polkadot_node_core_pvf::Priority, _execute_priority: PvfExecPriority, ) -> Result { unreachable!() @@ -1296,7 +1296,7 @@ impl ValidationBackend for MockHeadsUp { _pvf: PvfPrepData, _timeout: Duration, _encoded_params: Vec, - _prepare_priority: polkadot_node_core_pvf::PreparePriority, + _prepare_priority: polkadot_node_core_pvf::Priority, _execute_priority: PvfExecPriority, ) -> Result { unreachable!() diff --git a/polkadot/node/core/pvf/src/host.rs b/polkadot/node/core/pvf/src/host.rs index 78c8d67c451b3..c145327573638 100644 --- a/polkadot/node/core/pvf/src/host.rs +++ b/polkadot/node/core/pvf/src/host.rs @@ -24,7 +24,7 @@ use crate::{ artifacts::{ArtifactId, ArtifactPathId, ArtifactState, Artifacts, ArtifactsCleanupConfig}, execute::{self, PendingExecutionRequest}, metrics::Metrics, - prepare, PreparePriority, SecurityStatus, ValidationError, LOG_TARGET, + prepare, Priority, SecurityStatus, ValidationError, LOG_TARGET, }; use always_assert::never; use futures::{ @@ -109,7 +109,7 @@ impl ValidationHost { pvf: PvfPrepData, exec_timeout: Duration, params: Vec, - priority: PreparePriority, + priority: Priority, execute_priority: PvfExecPriority, result_tx: ResultSender, ) -> Result<(), String> { @@ -150,7 +150,7 @@ struct ExecutePvfInputs { pvf: PvfPrepData, exec_timeout: Duration, params: Vec, - priority: PreparePriority, + priority: Priority, execute_priority: PvfExecPriority, result_tx: ResultSender, } @@ -517,11 +517,8 @@ async fn handle_precheck_pvf( } } else { artifacts.insert_preparing(artifact_id, vec![result_sender]); - send_prepare( - prepare_queue, - prepare::ToQueue::Enqueue { priority: PreparePriority::Normal, pvf }, - ) - .await?; + send_prepare(prepare_queue, prepare::ToQueue::Enqueue { priority: Priority::Normal, pvf }) + .await?; } Ok(()) } @@ -716,7 +713,7 @@ async fn handle_heads_up( send_prepare( prepare_queue, prepare::ToQueue::Enqueue { - priority: PreparePriority::Normal, + priority: Priority::Normal, pvf: active_pvf, }, ) @@ -730,7 +727,7 @@ async fn handle_heads_up( send_prepare( prepare_queue, - prepare::ToQueue::Enqueue { priority: PreparePriority::Normal, pvf: active_pvf }, + prepare::ToQueue::Enqueue { priority: Priority::Normal, pvf: active_pvf }, ) .await?; } @@ -876,7 +873,7 @@ async fn enqueue_prepare_for_execute( prepare_queue: &mut mpsc::Sender, awaiting_prepare: &mut AwaitingPrepare, pvf: PvfPrepData, - priority: PreparePriority, + priority: Priority, artifact_id: ArtifactId, pending_execution_request: PendingExecutionRequest, ) -> Result<(), Fatal> { @@ -1257,7 +1254,7 @@ pub(crate) mod tests { PvfPrepData::from_discriminator(1), TEST_EXECUTION_TIMEOUT, b"pvf1".to_vec(), - PreparePriority::Normal, + Priority::Normal, PvfExecPriority::Backing, result_tx, ) @@ -1269,7 +1266,7 @@ pub(crate) mod tests { PvfPrepData::from_discriminator(1), TEST_EXECUTION_TIMEOUT, b"pvf1".to_vec(), - PreparePriority::Critical, + Priority::Critical, PvfExecPriority::Backing, result_tx, ) @@ -1281,7 +1278,7 @@ pub(crate) mod tests { PvfPrepData::from_discriminator(2), TEST_EXECUTION_TIMEOUT, b"pvf2".to_vec(), - PreparePriority::Normal, + Priority::Normal, PvfExecPriority::Backing, result_tx, ) @@ -1423,7 +1420,7 @@ pub(crate) mod tests { PvfPrepData::from_discriminator(1), TEST_EXECUTION_TIMEOUT, b"pvf2".to_vec(), - PreparePriority::Critical, + Priority::Critical, PvfExecPriority::Backing, result_tx, ) @@ -1471,7 +1468,7 @@ pub(crate) mod tests { PvfPrepData::from_discriminator(2), TEST_EXECUTION_TIMEOUT, b"pvf2".to_vec(), - PreparePriority::Critical, + Priority::Critical, PvfExecPriority::Backing, result_tx, ) @@ -1574,7 +1571,7 @@ pub(crate) mod tests { PvfPrepData::from_discriminator(1), TEST_EXECUTION_TIMEOUT, b"pvf".to_vec(), - PreparePriority::Critical, + Priority::Critical, PvfExecPriority::Backing, result_tx, ) @@ -1605,7 +1602,7 @@ pub(crate) mod tests { PvfPrepData::from_discriminator(1), TEST_EXECUTION_TIMEOUT, b"pvf".to_vec(), - PreparePriority::Critical, + Priority::Critical, PvfExecPriority::Backing, result_tx_2, ) @@ -1628,7 +1625,7 @@ pub(crate) mod tests { PvfPrepData::from_discriminator(1), TEST_EXECUTION_TIMEOUT, b"pvf".to_vec(), - PreparePriority::Critical, + Priority::Critical, PvfExecPriority::Backing, result_tx_3, ) @@ -1679,7 +1676,7 @@ pub(crate) mod tests { PvfPrepData::from_discriminator(1), TEST_EXECUTION_TIMEOUT, b"pvf".to_vec(), - PreparePriority::Critical, + Priority::Critical, PvfExecPriority::Backing, result_tx, ) @@ -1710,7 +1707,7 @@ pub(crate) mod tests { PvfPrepData::from_discriminator(1), TEST_EXECUTION_TIMEOUT, b"pvf".to_vec(), - PreparePriority::Critical, + Priority::Critical, PvfExecPriority::Backing, result_tx_2, ) @@ -1733,7 +1730,7 @@ pub(crate) mod tests { PvfPrepData::from_discriminator(1), TEST_EXECUTION_TIMEOUT, b"pvf".to_vec(), - PreparePriority::Critical, + Priority::Critical, PvfExecPriority::Backing, result_tx_3, ) @@ -1800,7 +1797,7 @@ pub(crate) mod tests { PvfPrepData::from_discriminator(1), TEST_EXECUTION_TIMEOUT, b"pvf1".to_vec(), - PreparePriority::Normal, + Priority::Normal, PvfExecPriority::Backing, result_tx, ) diff --git a/polkadot/node/core/pvf/src/lib.rs b/polkadot/node/core/pvf/src/lib.rs index 1be9a9e12fd18..462498fa8f6b1 100644 --- a/polkadot/node/core/pvf/src/lib.rs +++ b/polkadot/node/core/pvf/src/lib.rs @@ -110,7 +110,7 @@ pub use host::{ PREPARE_BINARY_NAME, }; pub use metrics::Metrics; -pub use priority::PreparePriority; +pub use priority::Priority; pub use worker_interface::{framed_recv, framed_send, JOB_TIMEOUT_WALL_CLOCK_FACTOR}; // Re-export some common types. diff --git a/polkadot/node/core/pvf/src/prepare/queue.rs b/polkadot/node/core/pvf/src/prepare/queue.rs index dd9b66095178a..c7bfa2f3b21ba 100644 --- a/polkadot/node/core/pvf/src/prepare/queue.rs +++ b/polkadot/node/core/pvf/src/prepare/queue.rs @@ -17,7 +17,7 @@ //! A queue that handles requests for PVF preparation. use super::pool::{self, Worker}; -use crate::{artifacts::ArtifactId, metrics::Metrics, PreparePriority, LOG_TARGET}; +use crate::{artifacts::ArtifactId, metrics::Metrics, Priority, LOG_TARGET}; use always_assert::{always, never}; use futures::{channel::mpsc, stream::StreamExt as _, Future, SinkExt}; use polkadot_node_core_pvf_common::{error::PrepareResult, pvf::PvfPrepData}; @@ -36,7 +36,7 @@ pub enum ToQueue { /// /// Note that it is incorrect to enqueue the same PVF again without first receiving the /// [`FromQueue`] response. - Enqueue { priority: PreparePriority, pvf: PvfPrepData }, + Enqueue { priority: Priority, pvf: PvfPrepData }, } /// A response from queue. @@ -80,7 +80,7 @@ slotmap::new_key_type! { pub struct Job; } struct JobData { /// The priority of this job. Can be bumped. - priority: PreparePriority, + priority: Priority, pvf: PvfPrepData, worker: Option, } @@ -106,18 +106,18 @@ struct Unscheduled { } impl Unscheduled { - fn queue_mut(&mut self, prio: PreparePriority) -> &mut VecDeque { + fn queue_mut(&mut self, prio: Priority) -> &mut VecDeque { match prio { - PreparePriority::Normal => &mut self.normal, - PreparePriority::Critical => &mut self.critical, + Priority::Normal => &mut self.normal, + Priority::Critical => &mut self.critical, } } - fn add(&mut self, prio: PreparePriority, job: Job) { + fn add(&mut self, prio: Priority, job: Job) { self.queue_mut(prio).push_back(job); } - fn readd(&mut self, prio: PreparePriority, job: Job) { + fn readd(&mut self, prio: Priority, job: Job) { self.queue_mut(prio).push_front(job); } @@ -126,8 +126,8 @@ impl Unscheduled { } fn next(&mut self) -> Option { - let mut check = |prio: PreparePriority| self.queue_mut(prio).pop_front(); - check(PreparePriority::Critical).or_else(|| check(PreparePriority::Normal)) + let mut check = |prio: Priority| self.queue_mut(prio).pop_front(); + check(Priority::Critical).or_else(|| check(Priority::Normal)) } } @@ -219,7 +219,7 @@ async fn handle_to_queue(queue: &mut Queue, to_queue: ToQueue) -> Result<(), Fat async fn handle_enqueue( queue: &mut Queue, - priority: PreparePriority, + priority: Priority, pvf: PvfPrepData, ) -> Result<(), Fatal> { gum::debug!( @@ -394,7 +394,7 @@ async fn handle_worker_rip(queue: &mut Queue, worker: Worker) -> Result<(), Fata // this path cannot be hit; // qed. never!("the job of the ripped worker must be known but it is not"); - PreparePriority::Normal + Priority::Normal }); queue.unscheduled.readd(priority, job); } @@ -604,7 +604,7 @@ mod tests { async fn properly_concludes() { let mut test = Test::new(2, 2); - test.send_queue(ToQueue::Enqueue { priority: PreparePriority::Normal, pvf: pvf(1) }); + test.send_queue(ToQueue::Enqueue { priority: Priority::Normal, pvf: pvf(1) }); assert_eq!(test.poll_and_recv_to_pool().await, pool::ToPool::Spawn); let w = test.workers.insert(()); @@ -625,7 +625,7 @@ mod tests { async fn dont_spawn_over_soft_limit_unless_critical() { let mut test = Test::new(2, 3); - let priority = PreparePriority::Normal; + let priority = Priority::Normal; test.send_queue(ToQueue::Enqueue { priority, pvf: PvfPrepData::from_discriminator(1) }); test.send_queue(ToQueue::Enqueue { priority, pvf: PvfPrepData::from_discriminator(2) }); // Start a non-precheck preparation for this one. @@ -658,7 +658,7 @@ mod tests { // Enqueue a critical job. test.send_queue(ToQueue::Enqueue { - priority: PreparePriority::Critical, + priority: Priority::Critical, pvf: PvfPrepData::from_discriminator(4), }); @@ -672,7 +672,7 @@ mod tests { let mut test = Test::new(1, 2); test.send_queue(ToQueue::Enqueue { - priority: PreparePriority::Normal, + priority: Priority::Normal, pvf: PvfPrepData::from_discriminator(1), }); assert_eq!(test.poll_and_recv_to_pool().await, pool::ToPool::Spawn); @@ -682,7 +682,7 @@ mod tests { // Enqueue a critical job, which warrants spawning over the soft limit. test.send_queue(ToQueue::Enqueue { - priority: PreparePriority::Critical, + priority: Priority::Critical, pvf: PvfPrepData::from_discriminator(2), }); assert_eq!(test.poll_and_recv_to_pool().await, pool::ToPool::Spawn); @@ -706,7 +706,7 @@ mod tests { async fn worker_mass_die_out_doesnt_stall_queue() { let mut test = Test::new(2, 2); - let priority = PreparePriority::Normal; + let priority = Priority::Normal; test.send_queue(ToQueue::Enqueue { priority, pvf: PvfPrepData::from_discriminator(1) }); test.send_queue(ToQueue::Enqueue { priority, pvf: PvfPrepData::from_discriminator(2) }); // Start a non-precheck preparation for this one. @@ -748,7 +748,7 @@ mod tests { let mut test = Test::new(2, 2); test.send_queue(ToQueue::Enqueue { - priority: PreparePriority::Normal, + priority: Priority::Normal, pvf: PvfPrepData::from_discriminator(1), }); @@ -772,7 +772,7 @@ mod tests { let mut test = Test::new(2, 2); test.send_queue(ToQueue::Enqueue { - priority: PreparePriority::Normal, + priority: Priority::Normal, pvf: PvfPrepData::from_discriminator(1), }); diff --git a/polkadot/node/core/pvf/src/priority.rs b/polkadot/node/core/pvf/src/priority.rs index 89ca3aaa34b83..39bb47f4a65c3 100644 --- a/polkadot/node/core/pvf/src/priority.rs +++ b/polkadot/node/core/pvf/src/priority.rs @@ -18,7 +18,7 @@ use polkadot_node_subsystem::messages::PvfExecPriority; /// A priority assigned to preparation of a PVF. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] -pub enum PreparePriority { +pub enum Priority { /// Normal priority for things that do not require immediate response, but still need to be /// done pretty quick. /// @@ -31,20 +31,20 @@ pub enum PreparePriority { Critical, } -impl PreparePriority { +impl Priority { /// Returns `true` if `self` is `Critical` pub fn is_critical(self) -> bool { - self == PreparePriority::Critical + self == Priority::Critical } } -impl From for PreparePriority { +impl From for Priority { fn from(priority: PvfExecPriority) -> Self { match priority { - PvfExecPriority::Dispute => PreparePriority::Critical, - PvfExecPriority::Approval => PreparePriority::Critical, - PvfExecPriority::BackingSystemParas => PreparePriority::Normal, - PvfExecPriority::Backing => PreparePriority::Normal, + PvfExecPriority::Dispute => Priority::Critical, + PvfExecPriority::Approval => Priority::Critical, + PvfExecPriority::BackingSystemParas => Priority::Normal, + PvfExecPriority::Backing => Priority::Normal, } } } diff --git a/polkadot/node/core/pvf/tests/it/main.rs b/polkadot/node/core/pvf/tests/it/main.rs index 478bcd9086985..b173a982ffb26 100644 --- a/polkadot/node/core/pvf/tests/it/main.rs +++ b/polkadot/node/core/pvf/tests/it/main.rs @@ -124,7 +124,7 @@ impl TestHost { ), TEST_EXECUTION_TIMEOUT, params.encode(), - polkadot_node_core_pvf::PreparePriority::Normal, + polkadot_node_core_pvf::Priority::Normal, PvfExecPriority::Backing, result_tx, ) From e7223fefa6881aeae9932b97696a09e9d5171b51 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 25 Jul 2024 16:12:03 +0200 Subject: [PATCH 26/48] Fix docs --- polkadot/node/core/candidate-validation/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polkadot/node/core/candidate-validation/src/lib.rs b/polkadot/node/core/candidate-validation/src/lib.rs index 029be57fb5f6f..203765b5d0eb0 100644 --- a/polkadot/node/core/candidate-validation/src/lib.rs +++ b/polkadot/node/core/candidate-validation/src/lib.rs @@ -1019,7 +1019,7 @@ trait ValidationBackend { encoded_params: Vec, // The priority for the preparation job. prepare_priority: polkadot_node_core_pvf::Priority, - // The priority for the preparation job. + // The priority for the execution job. execute_priority: PvfExecPriority, ) -> Result; @@ -1168,7 +1168,7 @@ impl ValidationBackend for ValidationHost { encoded_params: Vec, // The priority for the preparation job. prepare_priority: polkadot_node_core_pvf::Priority, - // The priority for the preparation job. + // The priority for the execution job. execute_priority: PvfExecPriority, ) -> Result { let (tx, rx) = oneshot::channel(); From d3d55d6f7295bda969e6b3dd6db371277df92973 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 25 Jul 2024 16:59:33 +0200 Subject: [PATCH 27/48] Update logic and explanations --- polkadot/node/core/pvf/src/execute/queue.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/polkadot/node/core/pvf/src/execute/queue.rs b/polkadot/node/core/pvf/src/execute/queue.rs index 12b642f10d5f4..8d7722be8fefe 100644 --- a/polkadot/node/core/pvf/src/execute/queue.rs +++ b/polkadot/node/core/pvf/src/execute/queue.rs @@ -628,8 +628,15 @@ struct Unscheduled { } impl Unscheduled { - // A threshold reaching which we prioritize approval jobs + // A threshold reaching which we reset counted jobs. + // Max number of jobs per block assuming 6s window, 2 CPU cores, and 2s for a run. const MAX_COUNT: usize = 12; + // A threshold in percentages, the portion a current priority can "steal" from lower ones. + // For example: + // Disputes take 70%, leaving 30% for approvals and all backings. + // 80% of the remaining goes to approvals, which is 30% * 80% = 24% of the original 100%. + // If we used parts of the original 100%, approvals can't take more than 24%, + // even if there are no disputes. const FULFILLED_THRESHOLDS: &'static [(PvfExecPriority, usize)] = &[ (PvfExecPriority::Dispute, 70), (PvfExecPriority::Approval, 80), @@ -689,7 +696,9 @@ impl Unscheduled { return false } - count * 100 / total_count >= threshold + // Because we operate through a small range, we can't let a priority go over the + // threshold, so we check fulfillment by adding one more run + (count + 1) * 100 / total_count >= threshold } fn log(&mut self, priority: PvfExecPriority) { From dc78fecc4313b12954d213ba35f82712c1f66100 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 25 Jul 2024 17:29:54 +0200 Subject: [PATCH 28/48] Update logic --- polkadot/node/core/pvf/src/execute/queue.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/polkadot/node/core/pvf/src/execute/queue.rs b/polkadot/node/core/pvf/src/execute/queue.rs index 8d7722be8fefe..787633d5c9cda 100644 --- a/polkadot/node/core/pvf/src/execute/queue.rs +++ b/polkadot/node/core/pvf/src/execute/queue.rs @@ -656,7 +656,11 @@ impl Unscheduled { fn select_next_priority(&self) -> PvfExecPriority { PvfExecPriority::iter() .find(|priority| self.has_pending(priority) && !self.is_fulfilled(priority)) - .unwrap_or(PvfExecPriority::Backing) + .unwrap_or_else(|| { + PvfExecPriority::iter() + .find(|priority| self.has_pending(priority)) + .unwrap_or(PvfExecPriority::Backing) + }) } fn get_mut(&mut self, priority: PvfExecPriority) -> Option<&mut VecDeque> { @@ -785,5 +789,13 @@ mod tests { // Fulfill system parachains backing jobs unscheduled.log(BackingSystemParas); assert_eq!(unscheduled.select_next_priority(), Backing); + + // Leave only approval jobs which are fulfilled + unscheduled.reset_counter(); + unscheduled.get_mut(BackingSystemParas).unwrap().clear(); + unscheduled.get_mut(Backing).unwrap().clear(); + unscheduled.add(create_execution_job(), Approval); + unscheduled.log(Approval); + assert_eq!(unscheduled.select_next_priority(), Approval); } } From a9e628c6bea5aee6dd201d7f0d6938597e4a4d8a Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 25 Jul 2024 17:55:22 +0200 Subject: [PATCH 29/48] Add metrics --- polkadot/node/core/pvf/src/execute/queue.rs | 1 + polkadot/node/core/pvf/src/metrics.rs | 19 +++++++++++++++++++ polkadot/node/subsystem-types/src/messages.rs | 12 ++++++++++++ 3 files changed, 32 insertions(+) diff --git a/polkadot/node/core/pvf/src/execute/queue.rs b/polkadot/node/core/pvf/src/execute/queue.rs index 787633d5c9cda..dcc934ce4d5e0 100644 --- a/polkadot/node/core/pvf/src/execute/queue.rs +++ b/polkadot/node/core/pvf/src/execute/queue.rs @@ -280,6 +280,7 @@ impl Queue { } else { spawn_extra_worker(self, job); } + self.metrics.on_execute_priority(priority); self.unscheduled.log(priority); } } diff --git a/polkadot/node/core/pvf/src/metrics.rs b/polkadot/node/core/pvf/src/metrics.rs index bc8d300037fe8..65de04c07ae0b 100644 --- a/polkadot/node/core/pvf/src/metrics.rs +++ b/polkadot/node/core/pvf/src/metrics.rs @@ -18,6 +18,7 @@ use polkadot_node_core_pvf_common::prepare::MemoryStats; use polkadot_node_metrics::metrics::{self, prometheus}; +use polkadot_node_subsystem::messages::PvfExecPriority; /// Validation host metrics. #[derive(Default, Clone)] @@ -105,6 +106,13 @@ impl Metrics { .observe((memory_stats.peak_tracked_alloc / 1024) as f64); } } + + /// When preparation pipeline concluded working on an item. + pub(crate) fn on_execute_priority(&self, priority: PvfExecPriority) { + if let Some(metrics) = &self.0 { + metrics.execute_priority_selected.with_label_values(&[priority.as_str()]).inc(); + } + } } #[derive(Clone)] @@ -129,6 +137,7 @@ struct MetricsInner { preparation_max_resident: prometheus::Histogram, // Peak allocation value, tracked by tracking-allocator preparation_peak_tracked_allocation: prometheus::Histogram, + execute_priority_selected: prometheus::CounterVec, } impl metrics::Metrics for Metrics { @@ -323,6 +332,16 @@ impl metrics::Metrics for Metrics { )?, registry, )?, + execute_priority_selected: prometheus::register( + prometheus::CounterVec::new( + prometheus::Opts::new( + "polkadot_pvf_execute_priority_selected", + "The total number of selected execute priorities", + ), + &["priority"], + )?, + registry, + )?, }; Ok(Metrics(Some(inner))) } diff --git a/polkadot/node/subsystem-types/src/messages.rs b/polkadot/node/subsystem-types/src/messages.rs index 359520060d1f4..f2132edbdfdae 100644 --- a/polkadot/node/subsystem-types/src/messages.rs +++ b/polkadot/node/subsystem-types/src/messages.rs @@ -219,6 +219,18 @@ pub enum PvfExecPriority { Backing, } +impl PvfExecPriority { + /// Converts priority level to &str + pub fn as_str(&self) -> &str { + match *self { + Self::Dispute => "dispute", + Self::Approval => "approval", + Self::BackingSystemParas => "backing_system_paras", + Self::Backing => "backing", + } + } +} + impl From for PvfExecKind { fn from(exec: PvfExecPriority) -> Self { match exec { From 25170af748e6d98987969d2c3070cd0fd1af732d Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Tue, 30 Jul 2024 11:48:33 +0200 Subject: [PATCH 30/48] Update nits --- polkadot/node/core/pvf/src/execute/queue.rs | 23 +++++++++++-------- polkadot/node/subsystem-types/src/messages.rs | 1 + 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/polkadot/node/core/pvf/src/execute/queue.rs b/polkadot/node/core/pvf/src/execute/queue.rs index dcc934ce4d5e0..7ce6be60d0eda 100644 --- a/polkadot/node/core/pvf/src/execute/queue.rs +++ b/polkadot/node/core/pvf/src/execute/queue.rs @@ -225,12 +225,12 @@ impl Queue { /// If all the workers are busy or the queue is empty, it does nothing. /// Should be called every time a new job arrives to the queue or a job finishes. fn try_assign_next_job(&mut self, finished_worker: Option) { - // New jobs are always pushed to the tail of the queue; the one at its head is always - // the eldest one. - + // We always work at the same priority level let priority = self.unscheduled.select_next_priority(); let Some(queue) = self.unscheduled.get_mut(priority) else { return }; + // New jobs are always pushed to the tail of the queue based on their priority; + // the one at its head of each queue is always the eldest one. let eldest = if let Some(eldest) = queue.get(0) { eldest } else { return }; // By default, we're going to execute the eldest job on any worker slot available, even if @@ -281,7 +281,7 @@ impl Queue { spawn_extra_worker(self, job); } self.metrics.on_execute_priority(priority); - self.unscheduled.log(priority); + self.unscheduled.mark_scheduled(priority); } } @@ -630,7 +630,10 @@ struct Unscheduled { impl Unscheduled { // A threshold reaching which we reset counted jobs. - // Max number of jobs per block assuming 6s window, 2 CPU cores, and 2s for a run. + // The max expected queue_size in normal conditions, at the beginning of each block, + // a validator will submit jobs for at least vrf_module_samples(6) + 1 for backing + // the parachain candidate they are assigned to, there is some buffer added to cover + // for situations, where more work arrives in the queue. const MAX_COUNT: usize = 12; // A threshold in percentages, the portion a current priority can "steal" from lower ones. // For example: @@ -706,7 +709,7 @@ impl Unscheduled { (count + 1) * 100 / total_count >= threshold } - fn log(&mut self, priority: PvfExecPriority) { + fn mark_scheduled(&mut self, priority: PvfExecPriority) { let current_count: &mut usize = self.counter.entry(priority).or_default(); *current_count += 1; @@ -770,7 +773,7 @@ mod tests { assert_eq!(unscheduled.select_next_priority(), Dispute); // Fulfill dispute jobs - unscheduled.log(Dispute); + unscheduled.mark_scheduled(Dispute); assert_eq!(unscheduled.select_next_priority(), Approval); // Remove dispute jobs @@ -779,7 +782,7 @@ mod tests { assert_eq!(unscheduled.select_next_priority(), Approval); // Fulfill approval jobs - unscheduled.log(Approval); + unscheduled.mark_scheduled(Approval); assert_eq!(unscheduled.select_next_priority(), BackingSystemParas); // Remove approval jobs @@ -788,7 +791,7 @@ mod tests { assert_eq!(unscheduled.select_next_priority(), BackingSystemParas); // Fulfill system parachains backing jobs - unscheduled.log(BackingSystemParas); + unscheduled.mark_scheduled(BackingSystemParas); assert_eq!(unscheduled.select_next_priority(), Backing); // Leave only approval jobs which are fulfilled @@ -796,7 +799,7 @@ mod tests { unscheduled.get_mut(BackingSystemParas).unwrap().clear(); unscheduled.get_mut(Backing).unwrap().clear(); unscheduled.add(create_execution_job(), Approval); - unscheduled.log(Approval); + unscheduled.mark_scheduled(Approval); assert_eq!(unscheduled.select_next_priority(), Approval); } } diff --git a/polkadot/node/subsystem-types/src/messages.rs b/polkadot/node/subsystem-types/src/messages.rs index f2132edbdfdae..75cc34ab5219a 100644 --- a/polkadot/node/subsystem-types/src/messages.rs +++ b/polkadot/node/subsystem-types/src/messages.rs @@ -207,6 +207,7 @@ pub enum CandidateValidationMessage { /// Extends primitives::PvfExecKind, which is a runtime parameter we don't want to change, /// to separate and prioritize execution jobs by request type. +/// The order is important: we iterate through the values. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, EnumIter)] pub enum PvfExecPriority { /// For dispute requests From 78279e8407d26da080b28d8ea222dbeda83a91a7 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Tue, 30 Jul 2024 14:44:11 +0200 Subject: [PATCH 31/48] Update tests --- polkadot/node/core/pvf/src/execute/queue.rs | 127 ++++++++++++++------ 1 file changed, 87 insertions(+), 40 deletions(-) diff --git a/polkadot/node/core/pvf/src/execute/queue.rs b/polkadot/node/core/pvf/src/execute/queue.rs index 7ce6be60d0eda..1f38618fd5ec0 100644 --- a/polkadot/node/core/pvf/src/execute/queue.rs +++ b/polkadot/node/core/pvf/src/execute/queue.rs @@ -756,50 +756,97 @@ mod tests { } #[test] - fn test_unscheduled_select_next_priority() { + fn test_unscheduled_priority_distribution() { use PvfExecPriority::*; + let mut priorities = vec![]; + let mut unscheduled = Unscheduled::new(); + for _ in 0..Unscheduled::MAX_COUNT { + unscheduled.add(create_execution_job(), Dispute); + unscheduled.add(create_execution_job(), Approval); + unscheduled.add(create_execution_job(), BackingSystemParas); + unscheduled.add(create_execution_job(), Backing); + } + + for _ in 0..Unscheduled::MAX_COUNT { + let priority = unscheduled.select_next_priority(); + priorities.push(priority); + unscheduled.mark_scheduled(priority); + } + + assert_eq!(priorities.iter().filter(|v| **v == Dispute).count(), 8); + assert_eq!(priorities.iter().filter(|v| **v == Approval).count(), 3); + assert_eq!(priorities.iter().filter(|v| **v == BackingSystemParas).count(), 1); + } + + #[test] + fn test_unscheduled_priority_distribution_without_backing_system_paras() { + use PvfExecPriority::*; + + let mut priorities = vec![]; - // With empty counter - assert_eq!(unscheduled.select_next_priority(), Backing); + let mut unscheduled = Unscheduled::new(); + for _ in 0..Unscheduled::MAX_COUNT { + unscheduled.add(create_execution_job(), Dispute); + unscheduled.add(create_execution_job(), Approval); + unscheduled.add(create_execution_job(), Backing); + } + + for _ in 0..Unscheduled::MAX_COUNT { + let priority = unscheduled.select_next_priority(); + priorities.push(priority); + unscheduled.mark_scheduled(priority); + } + + assert_eq!(priorities.iter().filter(|v| **v == Dispute).count(), 8); + assert_eq!(priorities.iter().filter(|v| **v == Approval).count(), 3); + assert_eq!(priorities.iter().filter(|v| **v == Backing).count(), 1); + } + + #[test] + fn test_unscheduled_priority_distribution_without_disputes() { + use PvfExecPriority::*; + + let mut priorities = vec![]; + + let mut unscheduled = Unscheduled::new(); + for _ in 0..Unscheduled::MAX_COUNT { + unscheduled.add(create_execution_job(), Approval); + unscheduled.add(create_execution_job(), BackingSystemParas); + unscheduled.add(create_execution_job(), Backing); + } + + for _ in 0..Unscheduled::MAX_COUNT { + let priority = unscheduled.select_next_priority(); + priorities.push(priority); + unscheduled.mark_scheduled(priority); + } + + assert_eq!(priorities.iter().filter(|v| **v == Approval).count(), 9); + assert_eq!(priorities.iter().filter(|v| **v == BackingSystemParas).count(), 2); + assert_eq!(priorities.iter().filter(|v| **v == Backing).count(), 1); + } + + #[test] + fn test_unscheduled_priority_distribution_without_disputes_and_only_one_backing() { + use PvfExecPriority::*; + + let mut priorities = vec![]; + + let mut unscheduled = Unscheduled::new(); + for _ in 0..Unscheduled::MAX_COUNT { + unscheduled.add(create_execution_job(), Approval); + } unscheduled.add(create_execution_job(), Backing); - assert_eq!(unscheduled.select_next_priority(), Backing); - unscheduled.add(create_execution_job(), BackingSystemParas); - assert_eq!(unscheduled.select_next_priority(), BackingSystemParas); - unscheduled.add(create_execution_job(), Approval); - assert_eq!(unscheduled.select_next_priority(), Approval); - unscheduled.add(create_execution_job(), Dispute); - assert_eq!(unscheduled.select_next_priority(), Dispute); - - // Fulfill dispute jobs - unscheduled.mark_scheduled(Dispute); - assert_eq!(unscheduled.select_next_priority(), Approval); - - // Remove dispute jobs - unscheduled.reset_counter(); - unscheduled.get_mut(Dispute).unwrap().clear(); - assert_eq!(unscheduled.select_next_priority(), Approval); - - // Fulfill approval jobs - unscheduled.mark_scheduled(Approval); - assert_eq!(unscheduled.select_next_priority(), BackingSystemParas); - - // Remove approval jobs - unscheduled.reset_counter(); - unscheduled.get_mut(Approval).unwrap().clear(); - assert_eq!(unscheduled.select_next_priority(), BackingSystemParas); - - // Fulfill system parachains backing jobs - unscheduled.mark_scheduled(BackingSystemParas); - assert_eq!(unscheduled.select_next_priority(), Backing); - - // Leave only approval jobs which are fulfilled - unscheduled.reset_counter(); - unscheduled.get_mut(BackingSystemParas).unwrap().clear(); - unscheduled.get_mut(Backing).unwrap().clear(); - unscheduled.add(create_execution_job(), Approval); - unscheduled.mark_scheduled(Approval); - assert_eq!(unscheduled.select_next_priority(), Approval); + + for _ in 0..Unscheduled::MAX_COUNT { + let priority = unscheduled.select_next_priority(); + priorities.push(priority); + unscheduled.mark_scheduled(priority); + } + + assert_eq!(priorities.iter().filter(|v| **v == Approval).count(), 11); + assert_eq!(priorities.iter().filter(|v| **v == Backing).count(), 1); } } From 2b2fedfd0594c36ffc64452da28737da840bebc4 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Tue, 30 Jul 2024 14:44:32 +0200 Subject: [PATCH 32/48] Add debug --- polkadot/node/core/pvf/src/execute/queue.rs | 29 +++++++++++++++++---- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/polkadot/node/core/pvf/src/execute/queue.rs b/polkadot/node/core/pvf/src/execute/queue.rs index 1f38618fd5ec0..7e542d0f1122d 100644 --- a/polkadot/node/core/pvf/src/execute/queue.rs +++ b/polkadot/node/core/pvf/src/execute/queue.rs @@ -658,13 +658,21 @@ impl Unscheduled { } fn select_next_priority(&self) -> PvfExecPriority { - PvfExecPriority::iter() + let priority = PvfExecPriority::iter() .find(|priority| self.has_pending(priority) && !self.is_fulfilled(priority)) .unwrap_or_else(|| { PvfExecPriority::iter() .find(|priority| self.has_pending(priority)) .unwrap_or(PvfExecPriority::Backing) - }) + }); + + gum::debug!( + target: LOG_TARGET, + ?priority, + "Selected next execution priority", + ); + + priority } fn get_mut(&mut self, priority: PvfExecPriority) -> Option<&mut VecDeque> { @@ -704,9 +712,20 @@ impl Unscheduled { return false } - // Because we operate through a small range, we can't let a priority go over the - // threshold, so we check fulfillment by adding one more run - (count + 1) * 100 / total_count >= threshold + let is_fulfilled = count * 100 / total_count >= threshold; + + gum::debug!( + target: LOG_TARGET, + ?priority, + ?count, + ?total_count, + "Execution priority is {}fulfilled: {}/{}%", + if is_fulfilled {""} else {"not "}, + count * 100 / total_count, + threshold + ); + + is_fulfilled } fn mark_scheduled(&mut self, priority: PvfExecPriority) { From 69a1043e8b44769bc8d4187e21313f122fea10fa Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Wed, 31 Jul 2024 14:49:52 +0200 Subject: [PATCH 33/48] Show how ordering works in a testcase --- polkadot/node/core/pvf/src/execute/queue.rs | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/polkadot/node/core/pvf/src/execute/queue.rs b/polkadot/node/core/pvf/src/execute/queue.rs index 7e542d0f1122d..3b458ae72dbd9 100644 --- a/polkadot/node/core/pvf/src/execute/queue.rs +++ b/polkadot/node/core/pvf/src/execute/queue.rs @@ -868,4 +868,25 @@ mod tests { assert_eq!(priorities.iter().filter(|v| **v == Approval).count(), 11); assert_eq!(priorities.iter().filter(|v| **v == Backing).count(), 1); } + + #[test] + fn test_unscheduled_does_not_postpone_backing() { + use PvfExecPriority::*; + + let mut priorities = vec![]; + + let mut unscheduled = Unscheduled::new(); + for _ in 0..Unscheduled::MAX_COUNT { + unscheduled.add(create_execution_job(), Approval); + } + unscheduled.add(create_execution_job(), Backing); + + for _ in 0..Unscheduled::MAX_COUNT { + let priority = unscheduled.select_next_priority(); + priorities.push(priority); + unscheduled.mark_scheduled(priority); + } + + assert_eq!(&priorities[..4], &[Approval, Backing, Approval, Approval]); + } } From 880b1335ac9828ad5895eac0fbe4a3cb77af05c1 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Wed, 7 Aug 2024 10:07:16 +0200 Subject: [PATCH 34/48] Add nits --- polkadot/node/core/pvf/src/execute/queue.rs | 17 ++++++++++++----- polkadot/node/subsystem-types/src/messages.rs | 3 ++- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/polkadot/node/core/pvf/src/execute/queue.rs b/polkadot/node/core/pvf/src/execute/queue.rs index 3b458ae72dbd9..d4803bca2624a 100644 --- a/polkadot/node/core/pvf/src/execute/queue.rs +++ b/polkadot/node/core/pvf/src/execute/queue.rs @@ -658,6 +658,13 @@ impl Unscheduled { } fn select_next_priority(&self) -> PvfExecPriority { + gum::debug!( + target: LOG_TARGET, + unscheduled = ?self.unscheduled.iter().map(|(p, q)| (*p, q.len())).collect::>(), + counter = ?self.counter, + "Selecting next execution priority...", + ); + let priority = PvfExecPriority::iter() .find(|priority| self.has_pending(priority) && !self.is_fulfilled(priority)) .unwrap_or_else(|| { @@ -703,25 +710,25 @@ impl Unscheduled { let Some(threshold) = Self::fulfilled_threshold(priority) else { return false }; let Some(count) = self.counter.get(&priority) else { return false }; // Every time we iterate by lower level priorities - let total_count: usize = self + let total_scheduled_at_priority_or_lower: usize = self .counter .iter() .filter_map(|(p, c)| if *p >= *priority { Some(c) } else { None }) .sum(); - if total_count == 0 { + if total_scheduled_at_priority_or_lower == 0 { return false } - let is_fulfilled = count * 100 / total_count >= threshold; + let is_fulfilled = count * 100 / total_scheduled_at_priority_or_lower >= threshold; gum::debug!( target: LOG_TARGET, ?priority, ?count, - ?total_count, + ?total_scheduled_at_priority_or_lower, "Execution priority is {}fulfilled: {}/{}%", if is_fulfilled {""} else {"not "}, - count * 100 / total_count, + count * 100 / total_scheduled_at_priority_or_lower, threshold ); diff --git a/polkadot/node/subsystem-types/src/messages.rs b/polkadot/node/subsystem-types/src/messages.rs index 75cc34ab5219a..3d43f7d94f511 100644 --- a/polkadot/node/subsystem-types/src/messages.rs +++ b/polkadot/node/subsystem-types/src/messages.rs @@ -207,7 +207,8 @@ pub enum CandidateValidationMessage { /// Extends primitives::PvfExecKind, which is a runtime parameter we don't want to change, /// to separate and prioritize execution jobs by request type. -/// The order is important: we iterate through the values. +/// The order is important, because we iterate through the values and assume it is going from higher +/// to lowest priority. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, EnumIter)] pub enum PvfExecPriority { /// For dispute requests From ebcfffef8b9be38cc57dfb2baab648b636cfe226 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Thu, 22 Aug 2024 18:53:17 +0200 Subject: [PATCH 35/48] Fix errors after conflicts resolving --- polkadot/node/core/candidate-validation/src/lib.rs | 4 ++-- polkadot/node/core/pvf/src/host.rs | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/polkadot/node/core/candidate-validation/src/lib.rs b/polkadot/node/core/candidate-validation/src/lib.rs index 2616a0c2a386c..d15b9a4754bd5 100644 --- a/polkadot/node/core/candidate-validation/src/lib.rs +++ b/polkadot/node/core/candidate-validation/src/lib.rs @@ -871,7 +871,7 @@ async fn validate_candidate_exhaustive( exec_timeout, persisted_validation_data.clone(), pov, - exec_kind.into(), + exec_kind.into(), exec_kind, ) .await @@ -880,7 +880,7 @@ async fn validate_candidate_exhaustive( validation_backend .validate_candidate_with_retry( validation_code.0, - pvf_exec_timeout(&executor_params, exec_kind), + pvf_exec_timeout(&executor_params, exec_kind.into()), persisted_validation_data.clone(), pov, executor_params, diff --git a/polkadot/node/core/pvf/src/host.rs b/polkadot/node/core/pvf/src/host.rs index 7d216e33cf0e9..5707be3f5a3be 100644 --- a/polkadot/node/core/pvf/src/host.rs +++ b/polkadot/node/core/pvf/src/host.rs @@ -37,7 +37,6 @@ use polkadot_node_core_pvf_common::{ pvf::PvfPrepData, }; use polkadot_node_primitives::PoV; -use polkadot_node_subsystem::{SubsystemError, SubsystemResult}; use polkadot_node_subsystem::{messages::PvfExecPriority, SubsystemError, SubsystemResult}; use polkadot_parachain_primitives::primitives::ValidationResult; use polkadot_primitives::PersistedValidationData; From afab63e5a7b7e536b977379b213501bf18210665 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Mon, 26 Aug 2024 14:40:23 +0200 Subject: [PATCH 36/48] Fix errors after conflicts resolving --- .../core/candidate-validation/src/tests.rs | 109 ------------------ polkadot/node/core/pvf/src/execute/queue.rs | 13 ++- polkadot/node/core/pvf/tests/it/main.rs | 4 +- 3 files changed, 14 insertions(+), 112 deletions(-) diff --git a/polkadot/node/core/candidate-validation/src/tests.rs b/polkadot/node/core/candidate-validation/src/tests.rs index 0d44e5426aaf0..7ebb187872f5e 100644 --- a/polkadot/node/core/candidate-validation/src/tests.rs +++ b/polkadot/node/core/candidate-validation/src/tests.rs @@ -954,115 +954,6 @@ fn compressed_code_works() { assert_matches!(v, Ok(ValidationResult::Valid(_, _))); } -#[test] -fn code_decompression_failure_is_error() { - let validation_data = PersistedValidationData { max_pov_size: 1024, ..Default::default() }; - let pov = PoV { block_data: BlockData(vec![1; 32]) }; - let head_data = HeadData(vec![1, 1, 1]); - - let raw_code = vec![2u8; VALIDATION_CODE_BOMB_LIMIT + 1]; - let validation_code = - sp_maybe_compressed_blob::compress(&raw_code, VALIDATION_CODE_BOMB_LIMIT + 1) - .map(ValidationCode) - .unwrap(); - - let descriptor = make_valid_candidate_descriptor( - ParaId::from(1_u32), - dummy_hash(), - validation_data.hash(), - pov.hash(), - validation_code.hash(), - head_data.hash(), - dummy_hash(), - Sr25519Keyring::Alice, - ); - - let validation_result = WasmValidationResult { - head_data, - new_validation_code: None, - upward_messages: Default::default(), - horizontal_messages: Default::default(), - processed_downward_messages: 0, - hrmp_watermark: 0, - }; - - let candidate_receipt = CandidateReceipt { descriptor, commitments_hash: Hash::zero() }; - - let pool = TaskExecutor::new(); - let (_ctx, _ctx_handle) = polkadot_node_subsystem_test_helpers::make_subsystem_context::< - AllMessages, - _, - >(pool.clone()); - - let v = executor::block_on(validate_candidate_exhaustive( - MockValidateCandidateBackend::with_hardcoded_result(Ok(validation_result)), - validation_data, - validation_code, - candidate_receipt, - Arc::new(pov), - ExecutorParams::default(), - PvfExecPriority::Backing, - &Default::default(), - )); - - assert_matches!(v, Err(_)); -} - -#[test] -fn pov_decompression_failure_is_invalid() { - let validation_data = - PersistedValidationData { max_pov_size: POV_BOMB_LIMIT as u32, ..Default::default() }; - let head_data = HeadData(vec![1, 1, 1]); - - let raw_block_data = vec![2u8; POV_BOMB_LIMIT + 1]; - let pov = sp_maybe_compressed_blob::compress(&raw_block_data, POV_BOMB_LIMIT + 1) - .map(|raw| PoV { block_data: BlockData(raw) }) - .unwrap(); - - let validation_code = ValidationCode(vec![2; 16]); - - let descriptor = make_valid_candidate_descriptor( - ParaId::from(1_u32), - dummy_hash(), - validation_data.hash(), - pov.hash(), - validation_code.hash(), - head_data.hash(), - dummy_hash(), - Sr25519Keyring::Alice, - ); - - let validation_result = WasmValidationResult { - head_data, - new_validation_code: None, - upward_messages: Default::default(), - horizontal_messages: Default::default(), - processed_downward_messages: 0, - hrmp_watermark: 0, - }; - - let candidate_receipt = CandidateReceipt { descriptor, commitments_hash: Hash::zero() }; - - let pool = TaskExecutor::new(); - let (_ctx, _ctx_handle) = polkadot_node_subsystem_test_helpers::make_subsystem_context::< - AllMessages, - _, - >(pool.clone()); - - let v = executor::block_on(validate_candidate_exhaustive( - MockValidateCandidateBackend::with_hardcoded_result(Ok(validation_result)), - validation_data, - validation_code, - candidate_receipt, - Arc::new(pov), - ExecutorParams::default(), - PvfExecPriority::Backing, - &Default::default(), - )); - - assert_matches!(v, Ok(ValidationResult::Invalid(InvalidCandidate::PoVDecompressionFailure))); -} - struct MockPreCheckBackend { result: Result<(), PrepareError>, } diff --git a/polkadot/node/core/pvf/src/execute/queue.rs b/polkadot/node/core/pvf/src/execute/queue.rs index 1a2213d79133f..7da073c4eda1f 100644 --- a/polkadot/node/core/pvf/src/execute/queue.rs +++ b/polkadot/node/core/pvf/src/execute/queue.rs @@ -782,16 +782,27 @@ impl Unscheduled { #[cfg(test)] mod tests { + use polkadot_node_primitives::BlockData; + use sp_core::H256; + use super::*; use crate::testing::artifact_id; use std::time::Duration; fn create_execution_job() -> ExecuteJob { let (result_tx, _result_rx) = oneshot::channel(); + let pvd = Arc::new(PersistedValidationData { + parent_head: Default::default(), + relay_parent_number: 1u32, + relay_parent_storage_root: H256::default(), + max_pov_size: 4096 * 1024, + }); + let pov = Arc::new(PoV { block_data: BlockData(b"pov".to_vec()) }); ExecuteJob { artifact: ArtifactPathId { id: artifact_id(0), path: PathBuf::new() }, exec_timeout: Duration::from_secs(10), - params: vec![], + pvd, + pov, executor_params: ExecutorParams::default(), result_tx, waiting_since: Instant::now(), diff --git a/polkadot/node/core/pvf/tests/it/main.rs b/polkadot/node/core/pvf/tests/it/main.rs index 0e5b596d6c9ad..5c3a97b9ec049 100644 --- a/polkadot/node/core/pvf/tests/it/main.rs +++ b/polkadot/node/core/pvf/tests/it/main.rs @@ -25,11 +25,11 @@ use polkadot_node_core_pvf::{ ValidationHost, JOB_TIMEOUT_WALL_CLOCK_FACTOR, }; use polkadot_node_primitives::{PoV, POV_BOMB_LIMIT, VALIDATION_CODE_BOMB_LIMIT}; -use polkadot_parachain_primitives::primitives::{BlockData, ValidationParams, ValidationResult}; +use polkadot_node_subsystem::messages::PvfExecPriority; +use polkadot_parachain_primitives::primitives::{BlockData, ValidationResult}; use polkadot_primitives::{ ExecutorParam, ExecutorParams, PersistedValidationData, PvfExecKind, PvfPrepKind, }; -use polkadot_node_subsystem::messages::PvfExecPriority; use sp_core::H256; use std::{io::Write, sync::Arc, time::Duration}; From d9c118fa01eec1b5d724adef79b4c90d9cf1b304 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Wed, 4 Sep 2024 18:00:39 +0200 Subject: [PATCH 37/48] Fix imports --- polkadot/node/core/approval-voting/src/lib.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index cc1d6e3e5429c..e5c5ef7568aa8 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -41,8 +41,8 @@ use polkadot_node_subsystem::{ ApprovalVotingMessage, AssignmentCheckError, AssignmentCheckResult, AvailabilityRecoveryMessage, BlockDescription, CandidateValidationMessage, ChainApiMessage, ChainSelectionMessage, CheckedIndirectAssignment, CheckedIndirectSignedApprovalVote, - DisputeCoordinatorMessage, HighestApprovedAncestorBlock, PvfExecPriority, RuntimeApiMessage, - RuntimeApiRequest, + DisputeCoordinatorMessage, HighestApprovedAncestorBlock, PvfExecPriority, + RuntimeApiMessage, RuntimeApiRequest, }, overseer, FromOrchestra, OverseerSignal, SpawnedSubsystem, SubsystemError, SubsystemResult, SubsystemSender, @@ -56,8 +56,8 @@ use polkadot_node_subsystem_util::{ }; use polkadot_primitives::{ ApprovalVoteMultipleCandidates, ApprovalVotingParams, BlockNumber, CandidateHash, - CandidateIndex, CandidateReceipt, CoreIndex, ExecutorParams, GroupIndex, Hash, PvfExecKind, - SessionIndex, SessionInfo, ValidatorId, ValidatorIndex, ValidatorPair, ValidatorSignature, + CandidateIndex, CandidateReceipt, CoreIndex, ExecutorParams, GroupIndex, Hash, SessionIndex, + SessionInfo, ValidatorId, ValidatorIndex, ValidatorPair, ValidatorSignature, }; use sc_keystore::LocalKeystore; use sp_application_crypto::Pair; From 1478b1856ada9b02c2aee870a6b7c9ed4a6afc82 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 20 Sep 2024 16:57:20 +0200 Subject: [PATCH 38/48] Address comments --- .../node/core/candidate-validation/src/lib.rs | 2 +- polkadot/node/core/pvf/src/execute/queue.rs | 96 ++++++++++--------- 2 files changed, 54 insertions(+), 44 deletions(-) diff --git a/polkadot/node/core/candidate-validation/src/lib.rs b/polkadot/node/core/candidate-validation/src/lib.rs index 7f05ff823bbe6..92290daec0552 100644 --- a/polkadot/node/core/candidate-validation/src/lib.rs +++ b/polkadot/node/core/candidate-validation/src/lib.rs @@ -997,7 +997,7 @@ trait ValidationBackend { retry_delay: Duration, // The priority for the preparation job. prepare_priority: polkadot_node_core_pvf::Priority, - // The priority for the preparation job. + // The priority for the execution job. execute_priority: PvfExecPriority, ) -> Result { let prep_timeout = pvf_prep_timeout(&executor_params, PvfPrepKind::Prepare); diff --git a/polkadot/node/core/pvf/src/execute/queue.rs b/polkadot/node/core/pvf/src/execute/queue.rs index 7da073c4eda1f..d2d8d218531f0 100644 --- a/polkadot/node/core/pvf/src/execute/queue.rs +++ b/polkadot/node/core/pvf/src/execute/queue.rs @@ -660,19 +660,30 @@ struct Unscheduled { } impl Unscheduled { - // A threshold reaching which we reset counted jobs. - // The max expected queue_size in normal conditions, at the beginning of each block, - // a validator will submit jobs for at least vrf_module_samples(6) + 1 for backing - // the parachain candidate they are assigned to, there is some buffer added to cover - // for situations, where more work arrives in the queue. - const MAX_COUNT: usize = 12; - // A threshold in percentages, the portion a current priority can "steal" from lower ones. - // For example: - // Disputes take 70%, leaving 30% for approvals and all backings. - // 80% of the remaining goes to approvals, which is 30% * 80% = 24% of the original 100%. - // If we used parts of the original 100%, approvals can't take more than 24%, - // even if there are no disputes. - const FULFILLED_THRESHOLDS: &'static [(PvfExecPriority, usize)] = &[ + /// We keep track of every scheduled job in the `counter`, but reset it if the total number of + /// counted jobs reaches the threshold. This number is set as the maximum amount of jobs per + /// relay chain block possible with 4 CPU cores and 2 seconds of execution time. Under normal + /// conditions, the maximum expected queue size is at least vrf_module_samples(6) + 1 for + /// backing a parachain candidate. A buffer is added to cover situations where more work + /// arrives in the queue. + const SCHEDULING_WINDOW_SIZE: usize = 12; + + /// A threshold in percentages indicates how much time a current priority can "steal" from lower + /// priorities. Given the `SCHEDULING_WINDOW_SIZE` is 12 and all job priorities are present: + /// - Disputes consume 70% or 8 jobs in a row. + /// - The remaining 30% of original 100% is allocated for approval and all backing jobs. + /// - 80% or 3 jobs of the remaining goes to approvals. + /// - The remaining 6% of original 100% is allocated for all backing jobs. + /// - 100% or 1 job of the remaining goes to backing system parachains. + /// - Nothing is left for backing. + /// - The counter is restarted and the distribution starts from the beginning. + /// + /// This system might seem complex, but we operate with the remaining percentages because: + /// - Not all job types are present in each block. If we used parts of the original 100%, + /// approvals could not exceed 24%, even if there are no disputes. + /// - We cannot fully prioritize backing system parachains over backing other parachains based + /// on the distribution of the original 100%. + const PRIORITY_ALLOCATION_THRESHOLDS: &'static [(PvfExecPriority, usize)] = &[ (PvfExecPriority::Dispute, 70), (PvfExecPriority::Approval, 80), (PvfExecPriority::BackingSystemParas, 100), @@ -697,7 +708,7 @@ impl Unscheduled { ); let priority = PvfExecPriority::iter() - .find(|priority| self.has_pending(priority) && !self.is_fulfilled(priority)) + .find(|priority| self.has_pending(priority) && !self.has_reached_threshold(priority)) .unwrap_or_else(|| { PvfExecPriority::iter() .find(|priority| self.has_pending(priority)) @@ -725,20 +736,20 @@ impl Unscheduled { !self.unscheduled.get(priority).unwrap_or(&VecDeque::new()).is_empty() } - fn fulfilled_threshold(priority: &PvfExecPriority) -> Option { - Self::FULFILLED_THRESHOLDS.iter().find_map( - |&(p, value)| { - if p == *priority { - Some(value) - } else { - None - } - }, - ) + fn priority_allocation_threshold(priority: &PvfExecPriority) -> Option { + Self::PRIORITY_ALLOCATION_THRESHOLDS.iter().find_map(|&(p, value)| { + if p == *priority { + Some(value) + } else { + None + } + }) } - fn is_fulfilled(&self, priority: &PvfExecPriority) -> bool { - let Some(threshold) = Self::fulfilled_threshold(priority) else { return false }; + /// Checks if a given priority has reached its allocated threshold + /// The thresholds are defined in `PRIORITY_ALLOCATION_THRESHOLDS`. + fn has_reached_threshold(&self, priority: &PvfExecPriority) -> bool { + let Some(threshold) = Self::priority_allocation_threshold(priority) else { return false }; let Some(count) = self.counter.get(&priority) else { return false }; // Every time we iterate by lower level priorities let total_scheduled_at_priority_or_lower: usize = self @@ -750,27 +761,26 @@ impl Unscheduled { return false } - let is_fulfilled = count * 100 / total_scheduled_at_priority_or_lower >= threshold; + let has_reached_threshold = count * 100 / total_scheduled_at_priority_or_lower >= threshold; gum::debug!( target: LOG_TARGET, ?priority, ?count, ?total_scheduled_at_priority_or_lower, - "Execution priority is {}fulfilled: {}/{}%", - if is_fulfilled {""} else {"not "}, + "Execution priority has {}reached threshold: {}/{}%", + if has_reached_threshold {""} else {"not "}, count * 100 / total_scheduled_at_priority_or_lower, threshold ); - is_fulfilled + has_reached_threshold } fn mark_scheduled(&mut self, priority: PvfExecPriority) { - let current_count: &mut usize = self.counter.entry(priority).or_default(); - *current_count += 1; + *self.counter.entry(priority).or_default() += 1; - if self.counter.values().sum::() >= Self::MAX_COUNT { + if self.counter.values().sum::() >= Self::SCHEDULING_WINDOW_SIZE { self.reset_counter(); } } @@ -830,14 +840,14 @@ mod tests { let mut priorities = vec![]; let mut unscheduled = Unscheduled::new(); - for _ in 0..Unscheduled::MAX_COUNT { + for _ in 0..Unscheduled::SCHEDULING_WINDOW_SIZE { unscheduled.add(create_execution_job(), Dispute); unscheduled.add(create_execution_job(), Approval); unscheduled.add(create_execution_job(), BackingSystemParas); unscheduled.add(create_execution_job(), Backing); } - for _ in 0..Unscheduled::MAX_COUNT { + for _ in 0..Unscheduled::SCHEDULING_WINDOW_SIZE { let priority = unscheduled.select_next_priority(); priorities.push(priority); unscheduled.mark_scheduled(priority); @@ -855,13 +865,13 @@ mod tests { let mut priorities = vec![]; let mut unscheduled = Unscheduled::new(); - for _ in 0..Unscheduled::MAX_COUNT { + for _ in 0..Unscheduled::SCHEDULING_WINDOW_SIZE { unscheduled.add(create_execution_job(), Dispute); unscheduled.add(create_execution_job(), Approval); unscheduled.add(create_execution_job(), Backing); } - for _ in 0..Unscheduled::MAX_COUNT { + for _ in 0..Unscheduled::SCHEDULING_WINDOW_SIZE { let priority = unscheduled.select_next_priority(); priorities.push(priority); unscheduled.mark_scheduled(priority); @@ -879,13 +889,13 @@ mod tests { let mut priorities = vec![]; let mut unscheduled = Unscheduled::new(); - for _ in 0..Unscheduled::MAX_COUNT { + for _ in 0..Unscheduled::SCHEDULING_WINDOW_SIZE { unscheduled.add(create_execution_job(), Approval); unscheduled.add(create_execution_job(), BackingSystemParas); unscheduled.add(create_execution_job(), Backing); } - for _ in 0..Unscheduled::MAX_COUNT { + for _ in 0..Unscheduled::SCHEDULING_WINDOW_SIZE { let priority = unscheduled.select_next_priority(); priorities.push(priority); unscheduled.mark_scheduled(priority); @@ -903,12 +913,12 @@ mod tests { let mut priorities = vec![]; let mut unscheduled = Unscheduled::new(); - for _ in 0..Unscheduled::MAX_COUNT { + for _ in 0..Unscheduled::SCHEDULING_WINDOW_SIZE { unscheduled.add(create_execution_job(), Approval); } unscheduled.add(create_execution_job(), Backing); - for _ in 0..Unscheduled::MAX_COUNT { + for _ in 0..Unscheduled::SCHEDULING_WINDOW_SIZE { let priority = unscheduled.select_next_priority(); priorities.push(priority); unscheduled.mark_scheduled(priority); @@ -925,12 +935,12 @@ mod tests { let mut priorities = vec![]; let mut unscheduled = Unscheduled::new(); - for _ in 0..Unscheduled::MAX_COUNT { + for _ in 0..Unscheduled::SCHEDULING_WINDOW_SIZE { unscheduled.add(create_execution_job(), Approval); } unscheduled.add(create_execution_job(), Backing); - for _ in 0..Unscheduled::MAX_COUNT { + for _ in 0..Unscheduled::SCHEDULING_WINDOW_SIZE { let priority = unscheduled.select_next_priority(); priorities.push(priority); unscheduled.mark_scheduled(priority); From 08125422eec88e86cb14dfa38f8ff41e720ea104 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 20 Sep 2024 17:01:03 +0200 Subject: [PATCH 39/48] Update lockfile --- Cargo.lock | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index f98c399178603..6ba96d0930556 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -14518,6 +14518,7 @@ dependencies = [ "slotmap", "sp-core 28.0.0", "sp-maybe-compressed-blob 11.0.0", + "strum 0.26.2", "tempfile", "test-parachain-adder", "test-parachain-halt", @@ -14791,6 +14792,7 @@ dependencies = [ "sp-blockchain", "sp-consensus-babe", "sp-runtime 31.0.1", + "strum 0.26.2", "substrate-prometheus-endpoint", "thiserror", ] From b979bb551858630dcc8064166a1c171a48890c71 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 20 Sep 2024 17:08:49 +0200 Subject: [PATCH 40/48] Add prdoc --- prdoc/pr_4837.prdoc | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 prdoc/pr_4837.prdoc diff --git a/prdoc/pr_4837.prdoc b/prdoc/pr_4837.prdoc new file mode 100644 index 0000000000000..5ee2f85f76019 --- /dev/null +++ b/prdoc/pr_4837.prdoc @@ -0,0 +1,12 @@ +# Schema: Polkadot SDK PRDoc Schema (prdoc) v1.0.0 +# See doc at https://raw.githubusercontent.com/paritytech/polkadot-sdk/master/prdoc/schema_user.json + +title: Add PVF execution priority + +doc: + - audience: Node Dev + description: | + The new logic optimizes the distribution of execution jobs for disputes, approvals, and backings. + The main goal is to create back pressure for backing in the presence of disputes or numerous approval jobs. + +crates: [ ] From 54c2812e44eae521f870bad0a94a18fe8ecb0660 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 20 Sep 2024 17:12:47 +0200 Subject: [PATCH 41/48] Update prdoc --- prdoc/pr_4837.prdoc | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/prdoc/pr_4837.prdoc b/prdoc/pr_4837.prdoc index 5ee2f85f76019..20be0d28cf7b9 100644 --- a/prdoc/pr_4837.prdoc +++ b/prdoc/pr_4837.prdoc @@ -9,4 +9,18 @@ doc: The new logic optimizes the distribution of execution jobs for disputes, approvals, and backings. The main goal is to create back pressure for backing in the presence of disputes or numerous approval jobs. -crates: [ ] +crates: + - name: polkadot-overseer + bump: minor + - name: polkadot-node-subsystem-types + bump: minor + - name: polkadot-node-core-approval-voting + bump: minor + - name: polkadot-node-core-backing + bump: minor + - name: polkadot-node-core-candidate-validation + bump: minor + - name: polkadot-node-core-pvf + bump: minor + - name: polkadot-node-core-dispute-coordinator + bump: minor From 6ce2f5a977550004da4e860b50eb310e2508b362 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Fri, 20 Sep 2024 18:53:46 +0200 Subject: [PATCH 42/48] Update prdoc --- prdoc/pr_4837.prdoc | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/prdoc/pr_4837.prdoc b/prdoc/pr_4837.prdoc index 20be0d28cf7b9..55c12cc92a1c1 100644 --- a/prdoc/pr_4837.prdoc +++ b/prdoc/pr_4837.prdoc @@ -10,17 +10,17 @@ doc: The main goal is to create back pressure for backing in the presence of disputes or numerous approval jobs. crates: + - name: polkadot-node-core-pvf + bump: major - name: polkadot-overseer - bump: minor + bump: patch - name: polkadot-node-subsystem-types - bump: minor + bump: patch - name: polkadot-node-core-approval-voting - bump: minor + bump: patch - name: polkadot-node-core-backing - bump: minor + bump: patch - name: polkadot-node-core-candidate-validation - bump: minor - - name: polkadot-node-core-pvf - bump: minor + bump: patch - name: polkadot-node-core-dispute-coordinator - bump: minor + bump: patch From 7b577b51b45a6db9806e846278045661a55266e6 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Wed, 9 Oct 2024 16:04:35 +0200 Subject: [PATCH 43/48] Rename PvfExecPriority -> PvfExecKind --- polkadot/node/core/approval-voting/src/lib.rs | 4 +- .../node/core/approval-voting/src/tests.rs | 10 ++-- polkadot/node/core/backing/src/lib.rs | 6 +- polkadot/node/core/backing/src/tests/mod.rs | 20 +++---- .../src/tests/prospective_parachains.rs | 2 +- .../node/core/candidate-validation/src/lib.rs | 16 ++--- .../core/candidate-validation/src/tests.rs | 34 +++++------ .../src/participation/mod.rs | 4 +- .../src/participation/tests.rs | 12 ++-- polkadot/node/core/pvf/src/execute/queue.rs | 60 +++++++++---------- polkadot/node/core/pvf/src/host.rs | 30 +++++----- polkadot/node/core/pvf/src/metrics.rs | 4 +- polkadot/node/core/pvf/src/priority.rs | 14 ++--- polkadot/node/core/pvf/tests/it/main.rs | 4 +- .../node/overseer/examples/minimal-example.rs | 4 +- polkadot/node/overseer/src/tests.rs | 6 +- polkadot/node/subsystem-types/src/messages.rs | 27 +++++---- 17 files changed, 129 insertions(+), 128 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index adef252d74687..a7caceeb8e264 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -41,7 +41,7 @@ use polkadot_node_subsystem::{ ApprovalVotingMessage, AssignmentCheckError, AssignmentCheckResult, AvailabilityRecoveryMessage, BlockDescription, CandidateValidationMessage, ChainApiMessage, ChainSelectionMessage, CheckedIndirectAssignment, CheckedIndirectSignedApprovalVote, - DisputeCoordinatorMessage, HighestApprovedAncestorBlock, PvfExecPriority, + DisputeCoordinatorMessage, HighestApprovedAncestorBlock, PvfExecKind, RuntimeApiMessage, RuntimeApiRequest, }, overseer, FromOrchestra, OverseerSignal, SpawnedSubsystem, SubsystemError, SubsystemResult, @@ -3602,7 +3602,7 @@ async fn launch_approval< candidate_receipt: candidate.clone(), pov: available_data.pov, executor_params, - exec_kind: PvfExecPriority::Approval, + exec_kind: PvfExecKind::Approval, response_sender: val_tx, }) .await; diff --git a/polkadot/node/core/approval-voting/src/tests.rs b/polkadot/node/core/approval-voting/src/tests.rs index 86f38cceca252..65aa4f894c23e 100644 --- a/polkadot/node/core/approval-voting/src/tests.rs +++ b/polkadot/node/core/approval-voting/src/tests.rs @@ -3131,7 +3131,7 @@ async fn handle_double_assignment_import( exec_kind, response_sender, .. - }) if exec_kind == PvfExecPriority::Approval => { + }) if exec_kind == PvfExecKind::Approval => { response_sender.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) .unwrap(); } @@ -4078,7 +4078,7 @@ async fn handle_approval_on_max_coalesce_count( for _ in &candidate_indices { assert_matches!( overseer_recv(virtual_overseer).await, - AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive{exec_kind, response_sender, ..}) if exec_kind == PvfExecPriority::Approval => { + AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive{exec_kind, response_sender, ..}) if exec_kind == PvfExecKind::Approval => { response_sender.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) .unwrap(); } @@ -4142,7 +4142,7 @@ async fn handle_approval_on_max_wait_time( for _ in &candidate_indices { assert_matches!( overseer_recv(virtual_overseer).await, - AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive{exec_kind, response_sender, ..}) if exec_kind == PvfExecPriority::Approval => { + AllMessages::CandidateValidation(CandidateValidationMessage::ValidateFromExhaustive{exec_kind, response_sender, ..}) if exec_kind == PvfExecKind::Approval => { response_sender.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) .unwrap(); } @@ -4645,7 +4645,7 @@ fn subsystem_relaunches_approval_work_on_restart() { exec_kind, response_sender, .. - }) if exec_kind == PvfExecPriority::Approval => { + }) if exec_kind == PvfExecKind::Approval => { response_sender.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) .unwrap(); } @@ -4765,7 +4765,7 @@ fn subsystem_sends_pending_approvals_on_approval_restart() { exec_kind, response_sender, .. - }) if exec_kind == PvfExecPriority::Approval => { + }) if exec_kind == PvfExecKind::Approval => { response_sender.send(Ok(ValidationResult::Valid(Default::default(), Default::default()))) .unwrap(); } diff --git a/polkadot/node/core/backing/src/lib.rs b/polkadot/node/core/backing/src/lib.rs index 556b852867a47..4463fb34b5103 100644 --- a/polkadot/node/core/backing/src/lib.rs +++ b/polkadot/node/core/backing/src/lib.rs @@ -89,7 +89,7 @@ use polkadot_node_subsystem::{ AvailabilityDistributionMessage, AvailabilityStoreMessage, CanSecondRequest, CandidateBackingMessage, CandidateValidationMessage, CollatorProtocolMessage, HypotheticalCandidate, HypotheticalMembershipRequest, IntroduceSecondedCandidateRequest, - ProspectiveParachainsMessage, ProvisionableData, ProvisionerMessage, PvfExecPriority, + ProspectiveParachainsMessage, ProvisionableData, ProvisionerMessage, PvfExecKind, RuntimeApiMessage, RuntimeApiRequest, StatementDistributionMessage, StoreAvailableDataError, }, @@ -637,9 +637,9 @@ async fn request_candidate_validation( pov, executor_params, exec_kind: if is_system { - PvfExecPriority::BackingSystemParas + PvfExecKind::BackingSystemParas } else { - PvfExecPriority::Backing + PvfExecKind::Backing }, response_sender: tx, }) diff --git a/polkadot/node/core/backing/src/tests/mod.rs b/polkadot/node/core/backing/src/tests/mod.rs index 6ed7b9c6d9795..d9c1fc9499e50 100644 --- a/polkadot/node/core/backing/src/tests/mod.rs +++ b/polkadot/node/core/backing/src/tests/mod.rs @@ -434,7 +434,7 @@ async fn assert_validate_from_exhaustive( ) if validation_data == *assert_pvd && validation_code == *assert_validation_code && *pov == *assert_pov && &candidate_receipt.descriptor == assert_candidate.descriptor() && - exec_kind == PvfExecPriority::BackingSystemParas && + exec_kind == PvfExecKind::BackingSystemParas && candidate_receipt.commitments_hash == assert_candidate.commitments.hash() => { response_sender.send(Ok(ValidationResult::Valid( @@ -651,7 +651,7 @@ fn backing_works(#[case] elastic_scaling_mvp: bool) { ) if validation_data == pvd_ab && validation_code == validation_code_ab && *pov == pov_ab && &candidate_receipt.descriptor == candidate_a.descriptor() && - exec_kind == PvfExecPriority::BackingSystemParas && + exec_kind == PvfExecKind::BackingSystemParas && candidate_receipt.commitments_hash == candidate_a_commitments_hash => { response_sender.send(Ok( @@ -1287,7 +1287,7 @@ fn backing_works_while_validation_ongoing() { ) if validation_data == pvd_abc && validation_code == validation_code_abc && *pov == pov_abc && &candidate_receipt.descriptor == candidate_a.descriptor() && - exec_kind == PvfExecPriority::BackingSystemParas && + exec_kind == PvfExecKind::BackingSystemParas && candidate_a_commitments_hash == candidate_receipt.commitments_hash => { // we never validate the candidate. our local node @@ -1454,7 +1454,7 @@ fn backing_misbehavior_works() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_a && &candidate_receipt.descriptor == candidate_a.descriptor() && - exec_kind == PvfExecPriority::BackingSystemParas && + exec_kind == PvfExecKind::BackingSystemParas && candidate_a_commitments_hash == candidate_receipt.commitments_hash => { response_sender.send(Ok( @@ -1621,7 +1621,7 @@ fn backing_dont_second_invalid() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_block_a && &candidate_receipt.descriptor == candidate_a.descriptor() && - exec_kind == PvfExecPriority::BackingSystemParas && + exec_kind == PvfExecKind::BackingSystemParas && candidate_a.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Ok(ValidationResult::Invalid(InvalidCandidate::BadReturn))).unwrap(); @@ -1661,7 +1661,7 @@ fn backing_dont_second_invalid() { ) if validation_data == pvd_b && validation_code == validation_code_b && *pov == pov_block_b && &candidate_receipt.descriptor == candidate_b.descriptor() && - exec_kind == PvfExecPriority::BackingSystemParas && + exec_kind == PvfExecKind::BackingSystemParas && candidate_b.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Ok( @@ -1788,7 +1788,7 @@ fn backing_second_after_first_fails_works() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_a && &candidate_receipt.descriptor == candidate.descriptor() && - exec_kind == PvfExecPriority::BackingSystemParas && + exec_kind == PvfExecKind::BackingSystemParas && candidate.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Ok(ValidationResult::Invalid(InvalidCandidate::BadReturn))).unwrap(); @@ -1932,7 +1932,7 @@ fn backing_works_after_failed_validation() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_a && &candidate_receipt.descriptor == candidate.descriptor() && - exec_kind == PvfExecPriority::BackingSystemParas && + exec_kind == PvfExecKind::BackingSystemParas && candidate.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Err(ValidationFailed("Internal test error".into()))).unwrap(); @@ -2211,7 +2211,7 @@ fn retry_works() { ) if validation_data == pvd_a && validation_code == validation_code_a && *pov == pov_a && &candidate_receipt.descriptor == candidate.descriptor() && - exec_kind == PvfExecPriority::BackingSystemParas && + exec_kind == PvfExecKind::BackingSystemParas && candidate.commitments.hash() == candidate_receipt.commitments_hash ); virtual_overseer @@ -2753,7 +2753,7 @@ fn validator_ignores_statements_from_disabled_validators() { ) if validation_data == pvd && validation_code == expected_validation_code && *pov == expected_pov && &candidate_receipt.descriptor == candidate.descriptor() && - exec_kind == PvfExecPriority::BackingSystemParas && + exec_kind == PvfExecKind::BackingSystemParas && candidate_commitments_hash == candidate_receipt.commitments_hash => { response_sender.send(Ok( diff --git a/polkadot/node/core/backing/src/tests/prospective_parachains.rs b/polkadot/node/core/backing/src/tests/prospective_parachains.rs index 3d7d4df0f1a64..57b2fabd43b06 100644 --- a/polkadot/node/core/backing/src/tests/prospective_parachains.rs +++ b/polkadot/node/core/backing/src/tests/prospective_parachains.rs @@ -276,7 +276,7 @@ async fn assert_validate_seconded_candidate( &validation_code == assert_validation_code && &*pov == assert_pov && &candidate_receipt.descriptor == candidate.descriptor() && - exec_kind == PvfExecPriority::BackingSystemParas && + exec_kind == PvfExecKind::BackingSystemParas && candidate.commitments.hash() == candidate_receipt.commitments_hash => { response_sender.send(Ok(ValidationResult::Valid( diff --git a/polkadot/node/core/candidate-validation/src/lib.rs b/polkadot/node/core/candidate-validation/src/lib.rs index 92290daec0552..a07406f018673 100644 --- a/polkadot/node/core/candidate-validation/src/lib.rs +++ b/polkadot/node/core/candidate-validation/src/lib.rs @@ -31,7 +31,7 @@ use polkadot_node_primitives::{InvalidCandidate, PoV, ValidationResult}; use polkadot_node_subsystem::{ errors::RuntimeApiError, messages::{ - CandidateValidationMessage, PreCheckOutcome, PvfExecPriority, RuntimeApiMessage, + CandidateValidationMessage, PreCheckOutcome, PvfExecKind, RuntimeApiMessage, RuntimeApiRequest, ValidationFailed, }, overseer, FromOrchestra, OverseerSignal, SpawnedSubsystem, SubsystemError, SubsystemResult, @@ -772,7 +772,7 @@ async fn validate_from_chain_state( candidate_receipt: CandidateReceipt, pov: Arc, executor_params: ExecutorParams, - exec_kind: PvfExecPriority, + exec_kind: PvfExecKind, metrics: &Metrics, ) -> Result where @@ -828,7 +828,7 @@ async fn validate_candidate_exhaustive( candidate_receipt: CandidateReceipt, pov: Arc, executor_params: ExecutorParams, - exec_kind: PvfExecPriority, + exec_kind: PvfExecKind, metrics: &Metrics, ) -> Result { let _timer = metrics.time_validate_candidate_exhaustive(); @@ -856,7 +856,7 @@ async fn validate_candidate_exhaustive( let result = match exec_kind { // Retry is disabled to reduce the chance of nondeterministic blocks getting backed and // honest backers getting slashed. - PvfExecPriority::Backing | PvfExecPriority::BackingSystemParas => { + PvfExecKind::Backing | PvfExecKind::BackingSystemParas => { let prep_timeout = pvf_prep_timeout(&executor_params, PvfPrepKind::Prepare); let exec_timeout = pvf_exec_timeout(&executor_params, exec_kind.into()); let pvf = PvfPrepData::from_code( @@ -877,7 +877,7 @@ async fn validate_candidate_exhaustive( ) .await }, - PvfExecPriority::Approval | PvfExecPriority::Dispute => + PvfExecKind::Approval | PvfExecKind::Dispute => validation_backend .validate_candidate_with_retry( validation_code.0, @@ -976,7 +976,7 @@ trait ValidationBackend { // The priority for the preparation job. prepare_priority: polkadot_node_core_pvf::Priority, // The priority for the execution job. - execute_priority: PvfExecPriority, + execute_priority: PvfExecKind, ) -> Result; /// Tries executing a PVF. Will retry once if an error is encountered that may have @@ -998,7 +998,7 @@ trait ValidationBackend { // The priority for the preparation job. prepare_priority: polkadot_node_core_pvf::Priority, // The priority for the execution job. - execute_priority: PvfExecPriority, + execute_priority: PvfExecKind, ) -> Result { let prep_timeout = pvf_prep_timeout(&executor_params, PvfPrepKind::Prepare); // Construct the PVF a single time, since it is an expensive operation. Cloning it is cheap. @@ -1127,7 +1127,7 @@ impl ValidationBackend for ValidationHost { // The priority for the preparation job. prepare_priority: polkadot_node_core_pvf::Priority, // The priority for the execution job. - execute_priority: PvfExecPriority, + execute_priority: PvfExecKind, ) -> Result { let (tx, rx) = oneshot::channel(); if let Err(err) = self diff --git a/polkadot/node/core/candidate-validation/src/tests.rs b/polkadot/node/core/candidate-validation/src/tests.rs index be3ca33f4c335..d278e803085c0 100644 --- a/polkadot/node/core/candidate-validation/src/tests.rs +++ b/polkadot/node/core/candidate-validation/src/tests.rs @@ -17,7 +17,7 @@ use std::sync::atomic::{AtomicUsize, Ordering}; use super::*; -use crate::PvfExecPriority; +use crate::PvfExecKind; use assert_matches::assert_matches; use futures::executor; use polkadot_node_core_pvf::PrepareError; @@ -389,7 +389,7 @@ impl ValidationBackend for MockValidateCandidateBackend { _pvd: Arc, _pov: Arc, _prepare_priority: polkadot_node_core_pvf::Priority, - _execute_priority: PvfExecPriority, + _execute_priority: PvfExecKind, ) -> Result { // This is expected to panic if called more times than expected, indicating an error in the // test. @@ -462,7 +462,7 @@ fn candidate_validation_ok_is_ok() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecPriority::Backing, + PvfExecKind::Backing, &Default::default(), )) .unwrap(); @@ -514,7 +514,7 @@ fn candidate_validation_bad_return_is_invalid() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecPriority::Backing, + PvfExecKind::Backing, &Default::default(), )) .unwrap(); @@ -595,7 +595,7 @@ fn candidate_validation_one_ambiguous_error_is_valid() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecPriority::Approval, + PvfExecKind::Approval, &Default::default(), )) .unwrap(); @@ -636,7 +636,7 @@ fn candidate_validation_multiple_ambiguous_errors_is_invalid() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecPriority::Approval, + PvfExecKind::Approval, &Default::default(), )) .unwrap(); @@ -648,7 +648,7 @@ fn candidate_validation_multiple_ambiguous_errors_is_invalid() { #[test] fn candidate_validation_retry_internal_errors() { let v = candidate_validation_retry_on_error_helper( - PvfExecPriority::Approval, + PvfExecKind::Approval, vec![ Err(InternalValidationError::HostCommunication("foo".into()).into()), // Throw an AJD error, we should still retry again. @@ -666,7 +666,7 @@ fn candidate_validation_retry_internal_errors() { #[test] fn candidate_validation_dont_retry_internal_errors() { let v = candidate_validation_retry_on_error_helper( - PvfExecPriority::Backing, + PvfExecKind::Backing, vec![ Err(InternalValidationError::HostCommunication("foo".into()).into()), // Throw an AWD error, we should still retry again. @@ -683,7 +683,7 @@ fn candidate_validation_dont_retry_internal_errors() { #[test] fn candidate_validation_retry_panic_errors() { let v = candidate_validation_retry_on_error_helper( - PvfExecPriority::Approval, + PvfExecKind::Approval, vec![ Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::JobError("foo".into()))), // Throw an AWD error, we should still retry again. @@ -700,7 +700,7 @@ fn candidate_validation_retry_panic_errors() { #[test] fn candidate_validation_dont_retry_panic_errors() { let v = candidate_validation_retry_on_error_helper( - PvfExecPriority::Backing, + PvfExecKind::Backing, vec![ Err(ValidationError::PossiblyInvalid(PossiblyInvalidError::JobError("foo".into()))), // Throw an AWD error, we should still retry again. @@ -714,7 +714,7 @@ fn candidate_validation_dont_retry_panic_errors() { } fn candidate_validation_retry_on_error_helper( - exec_kind: PvfExecPriority, + exec_kind: PvfExecKind, mock_errors: Vec>, ) -> Result { let validation_data = PersistedValidationData { max_pov_size: 1024, ..Default::default() }; @@ -792,7 +792,7 @@ fn candidate_validation_timeout_is_internal_error() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecPriority::Backing, + PvfExecKind::Backing, &Default::default(), )); @@ -837,7 +837,7 @@ fn candidate_validation_commitment_hash_mismatch_is_invalid() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecPriority::Backing, + PvfExecKind::Backing, &Default::default(), )) .unwrap(); @@ -889,7 +889,7 @@ fn candidate_validation_code_mismatch_is_invalid() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecPriority::Backing, + PvfExecKind::Backing, &Default::default(), )) .unwrap(); @@ -946,7 +946,7 @@ fn compressed_code_works() { candidate_receipt, Arc::new(pov), ExecutorParams::default(), - PvfExecPriority::Backing, + PvfExecKind::Backing, &Default::default(), )); @@ -972,7 +972,7 @@ impl ValidationBackend for MockPreCheckBackend { _pvd: Arc, _pov: Arc, _prepare_priority: polkadot_node_core_pvf::Priority, - _execute_priority: PvfExecPriority, + _execute_priority: PvfExecKind, ) -> Result { unreachable!() } @@ -1127,7 +1127,7 @@ impl ValidationBackend for MockHeadsUp { _pvd: Arc, _pov: Arc, _prepare_priority: polkadot_node_core_pvf::Priority, - _execute_priority: PvfExecPriority, + _execute_priority: PvfExecKind, ) -> Result { unreachable!() } diff --git a/polkadot/node/core/dispute-coordinator/src/participation/mod.rs b/polkadot/node/core/dispute-coordinator/src/participation/mod.rs index ccb7c8e42a8bd..2220f65e20a7c 100644 --- a/polkadot/node/core/dispute-coordinator/src/participation/mod.rs +++ b/polkadot/node/core/dispute-coordinator/src/participation/mod.rs @@ -27,7 +27,7 @@ use futures_timer::Delay; use polkadot_node_primitives::ValidationResult; use polkadot_node_subsystem::{ - messages::{AvailabilityRecoveryMessage, CandidateValidationMessage, PvfExecPriority}, + messages::{AvailabilityRecoveryMessage, CandidateValidationMessage, PvfExecKind}, overseer, ActiveLeavesUpdate, RecoveryError, }; use polkadot_node_subsystem_util::runtime::get_validation_code_by_hash; @@ -385,7 +385,7 @@ async fn participate( candidate_receipt: req.candidate_receipt().clone(), pov: available_data.pov, executor_params: req.executor_params(), - exec_kind: PvfExecPriority::Dispute, + exec_kind: PvfExecKind::Dispute, response_sender: validation_tx, }) .await; diff --git a/polkadot/node/core/dispute-coordinator/src/participation/tests.rs b/polkadot/node/core/dispute-coordinator/src/participation/tests.rs index 0a2c46c31f1d5..a6ab6f16df05e 100644 --- a/polkadot/node/core/dispute-coordinator/src/participation/tests.rs +++ b/polkadot/node/core/dispute-coordinator/src/participation/tests.rs @@ -26,8 +26,8 @@ use codec::Encode; use polkadot_node_primitives::{AvailableData, BlockData, InvalidCandidate, PoV}; use polkadot_node_subsystem::{ messages::{ - AllMessages, ChainApiMessage, DisputeCoordinatorMessage, PvfExecPriority, - RuntimeApiMessage, RuntimeApiRequest, + AllMessages, ChainApiMessage, DisputeCoordinatorMessage, PvfExecKind, RuntimeApiMessage, + RuntimeApiRequest, }, ActiveLeavesUpdate, SpawnGlue, }; @@ -116,7 +116,7 @@ pub async fn participation_full_happy_path( ctx_handle.recv().await, AllMessages::CandidateValidation( CandidateValidationMessage::ValidateFromExhaustive { candidate_receipt, exec_kind, response_sender, .. } - ) if exec_kind == PvfExecPriority::Dispute => { + ) if exec_kind == PvfExecKind::Dispute => { if expected_commitments_hash != candidate_receipt.commitments_hash { response_sender.send(Ok(ValidationResult::Invalid(InvalidCandidate::CommitmentsHashMismatch))).unwrap(); } else { @@ -450,7 +450,7 @@ fn cast_invalid_vote_if_validation_fails_or_is_invalid() { ctx_handle.recv().await, AllMessages::CandidateValidation( CandidateValidationMessage::ValidateFromExhaustive { exec_kind, response_sender, .. } - ) if exec_kind == PvfExecPriority::Dispute => { + ) if exec_kind == PvfExecKind::Dispute => { response_sender.send(Ok(ValidationResult::Invalid(InvalidCandidate::Timeout))).unwrap(); }, "overseer did not receive candidate validation message", @@ -487,7 +487,7 @@ fn cast_invalid_vote_if_commitments_dont_match() { ctx_handle.recv().await, AllMessages::CandidateValidation( CandidateValidationMessage::ValidateFromExhaustive { exec_kind, response_sender, .. } - ) if exec_kind == PvfExecPriority::Dispute => { + ) if exec_kind == PvfExecKind::Dispute => { response_sender.send(Ok(ValidationResult::Invalid(InvalidCandidate::CommitmentsHashMismatch))).unwrap(); }, "overseer did not receive candidate validation message", @@ -524,7 +524,7 @@ fn cast_valid_vote_if_validation_passes() { ctx_handle.recv().await, AllMessages::CandidateValidation( CandidateValidationMessage::ValidateFromExhaustive { exec_kind, response_sender, .. } - ) if exec_kind == PvfExecPriority::Dispute => { + ) if exec_kind == PvfExecKind::Dispute => { response_sender.send(Ok(ValidationResult::Valid(dummy_candidate_commitments(None), PersistedValidationData::default()))).unwrap(); }, "overseer did not receive candidate validation message", diff --git a/polkadot/node/core/pvf/src/execute/queue.rs b/polkadot/node/core/pvf/src/execute/queue.rs index d2d8d218531f0..dda094f2d075e 100644 --- a/polkadot/node/core/pvf/src/execute/queue.rs +++ b/polkadot/node/core/pvf/src/execute/queue.rs @@ -35,7 +35,7 @@ use polkadot_node_core_pvf_common::{ SecurityStatus, }; use polkadot_node_primitives::PoV; -use polkadot_node_subsystem::messages::PvfExecPriority; +use polkadot_node_subsystem::messages::PvfExecKind; use polkadot_primitives::{ExecutorParams, ExecutorParamsHash, PersistedValidationData}; use slotmap::HopSlotMap; use std::{ @@ -76,7 +76,7 @@ pub struct PendingExecutionRequest { pub pov: Arc, pub executor_params: ExecutorParams, pub result_tx: ResultSender, - pub execute_priority: PvfExecPriority, + pub execute_priority: PvfExecKind, } struct ExecuteJob { @@ -655,8 +655,8 @@ pub fn start( } struct Unscheduled { - unscheduled: HashMap>, - counter: HashMap, + unscheduled: HashMap>, + counter: HashMap, } impl Unscheduled { @@ -683,36 +683,36 @@ impl Unscheduled { /// approvals could not exceed 24%, even if there are no disputes. /// - We cannot fully prioritize backing system parachains over backing other parachains based /// on the distribution of the original 100%. - const PRIORITY_ALLOCATION_THRESHOLDS: &'static [(PvfExecPriority, usize)] = &[ - (PvfExecPriority::Dispute, 70), - (PvfExecPriority::Approval, 80), - (PvfExecPriority::BackingSystemParas, 100), - (PvfExecPriority::Backing, 100), + const PRIORITY_ALLOCATION_THRESHOLDS: &'static [(PvfExecKind, usize)] = &[ + (PvfExecKind::Dispute, 70), + (PvfExecKind::Approval, 80), + (PvfExecKind::BackingSystemParas, 100), + (PvfExecKind::Backing, 100), ]; fn new() -> Self { Self { - unscheduled: PvfExecPriority::iter() + unscheduled: PvfExecKind::iter() .map(|priority| (priority, VecDeque::new())) .collect(), - counter: PvfExecPriority::iter().map(|priority| (priority, 0)).collect(), + counter: PvfExecKind::iter().map(|priority| (priority, 0)).collect(), } } - fn select_next_priority(&self) -> PvfExecPriority { + fn select_next_priority(&self) -> PvfExecKind { gum::debug!( target: LOG_TARGET, - unscheduled = ?self.unscheduled.iter().map(|(p, q)| (*p, q.len())).collect::>(), + unscheduled = ?self.unscheduled.iter().map(|(p, q)| (*p, q.len())).collect::>(), counter = ?self.counter, "Selecting next execution priority...", ); - let priority = PvfExecPriority::iter() + let priority = PvfExecKind::iter() .find(|priority| self.has_pending(priority) && !self.has_reached_threshold(priority)) .unwrap_or_else(|| { - PvfExecPriority::iter() + PvfExecKind::iter() .find(|priority| self.has_pending(priority)) - .unwrap_or(PvfExecPriority::Backing) + .unwrap_or(PvfExecKind::Backing) }); gum::debug!( @@ -724,19 +724,19 @@ impl Unscheduled { priority } - fn get_mut(&mut self, priority: PvfExecPriority) -> Option<&mut VecDeque> { + fn get_mut(&mut self, priority: PvfExecKind) -> Option<&mut VecDeque> { self.unscheduled.get_mut(&priority) } - fn add(&mut self, job: ExecuteJob, priority: PvfExecPriority) { + fn add(&mut self, job: ExecuteJob, priority: PvfExecKind) { self.unscheduled.entry(priority).or_default().push_back(job); } - fn has_pending(&self, priority: &PvfExecPriority) -> bool { + fn has_pending(&self, priority: &PvfExecKind) -> bool { !self.unscheduled.get(priority).unwrap_or(&VecDeque::new()).is_empty() } - fn priority_allocation_threshold(priority: &PvfExecPriority) -> Option { + fn priority_allocation_threshold(priority: &PvfExecKind) -> Option { Self::PRIORITY_ALLOCATION_THRESHOLDS.iter().find_map(|&(p, value)| { if p == *priority { Some(value) @@ -748,7 +748,7 @@ impl Unscheduled { /// Checks if a given priority has reached its allocated threshold /// The thresholds are defined in `PRIORITY_ALLOCATION_THRESHOLDS`. - fn has_reached_threshold(&self, priority: &PvfExecPriority) -> bool { + fn has_reached_threshold(&self, priority: &PvfExecKind) -> bool { let Some(threshold) = Self::priority_allocation_threshold(priority) else { return false }; let Some(count) = self.counter.get(&priority) else { return false }; // Every time we iterate by lower level priorities @@ -777,7 +777,7 @@ impl Unscheduled { has_reached_threshold } - fn mark_scheduled(&mut self, priority: PvfExecPriority) { + fn mark_scheduled(&mut self, priority: PvfExecKind) { *self.counter.entry(priority).or_default() += 1; if self.counter.values().sum::() >= Self::SCHEDULING_WINDOW_SIZE { @@ -786,7 +786,7 @@ impl Unscheduled { } fn reset_counter(&mut self) { - self.counter = PvfExecPriority::iter().map(|kind| (kind, 0)).collect(); + self.counter = PvfExecKind::iter().map(|kind| (kind, 0)).collect(); } } @@ -823,11 +823,11 @@ mod tests { fn test_unscheduled_add() { let mut unscheduled = Unscheduled::new(); - PvfExecPriority::iter().for_each(|priority| { + PvfExecKind::iter().for_each(|priority| { unscheduled.add(create_execution_job(), priority); }); - PvfExecPriority::iter().for_each(|priority| { + PvfExecKind::iter().for_each(|priority| { let queue = unscheduled.unscheduled.get(&priority).unwrap(); assert_eq!(queue.len(), 1); }); @@ -835,7 +835,7 @@ mod tests { #[test] fn test_unscheduled_priority_distribution() { - use PvfExecPriority::*; + use PvfExecKind::*; let mut priorities = vec![]; @@ -860,7 +860,7 @@ mod tests { #[test] fn test_unscheduled_priority_distribution_without_backing_system_paras() { - use PvfExecPriority::*; + use PvfExecKind::*; let mut priorities = vec![]; @@ -884,7 +884,7 @@ mod tests { #[test] fn test_unscheduled_priority_distribution_without_disputes() { - use PvfExecPriority::*; + use PvfExecKind::*; let mut priorities = vec![]; @@ -908,7 +908,7 @@ mod tests { #[test] fn test_unscheduled_priority_distribution_without_disputes_and_only_one_backing() { - use PvfExecPriority::*; + use PvfExecKind::*; let mut priorities = vec![]; @@ -930,7 +930,7 @@ mod tests { #[test] fn test_unscheduled_does_not_postpone_backing() { - use PvfExecPriority::*; + use PvfExecKind::*; let mut priorities = vec![]; diff --git a/polkadot/node/core/pvf/src/host.rs b/polkadot/node/core/pvf/src/host.rs index 5707be3f5a3be..df12311153bc2 100644 --- a/polkadot/node/core/pvf/src/host.rs +++ b/polkadot/node/core/pvf/src/host.rs @@ -37,7 +37,7 @@ use polkadot_node_core_pvf_common::{ pvf::PvfPrepData, }; use polkadot_node_primitives::PoV; -use polkadot_node_subsystem::{messages::PvfExecPriority, SubsystemError, SubsystemResult}; +use polkadot_node_subsystem::{messages::PvfExecKind, SubsystemError, SubsystemResult}; use polkadot_parachain_primitives::primitives::ValidationResult; use polkadot_primitives::PersistedValidationData; use std::{ @@ -114,7 +114,7 @@ impl ValidationHost { pvd: Arc, pov: Arc, priority: Priority, - execute_priority: PvfExecPriority, + execute_priority: PvfExecKind, result_tx: ResultSender, ) -> Result<(), String> { self.to_host_tx @@ -157,7 +157,7 @@ struct ExecutePvfInputs { pvd: Arc, pov: Arc, priority: Priority, - execute_priority: PvfExecPriority, + execute_priority: PvfExecKind, result_tx: ResultSender, } @@ -1279,7 +1279,7 @@ pub(crate) mod tests { pvd.clone(), pov1.clone(), Priority::Normal, - PvfExecPriority::Backing, + PvfExecKind::Backing, result_tx, ) .await @@ -1292,7 +1292,7 @@ pub(crate) mod tests { pvd.clone(), pov1, Priority::Critical, - PvfExecPriority::Backing, + PvfExecKind::Backing, result_tx, ) .await @@ -1305,7 +1305,7 @@ pub(crate) mod tests { pvd, pov2, Priority::Normal, - PvfExecPriority::Backing, + PvfExecKind::Backing, result_tx, ) .await @@ -1455,7 +1455,7 @@ pub(crate) mod tests { pvd.clone(), pov.clone(), Priority::Critical, - PvfExecPriority::Backing, + PvfExecKind::Backing, result_tx, ) .await @@ -1504,7 +1504,7 @@ pub(crate) mod tests { pvd, pov, Priority::Critical, - PvfExecPriority::Backing, + PvfExecKind::Backing, result_tx, ) .await @@ -1615,7 +1615,7 @@ pub(crate) mod tests { pvd.clone(), pov.clone(), Priority::Critical, - PvfExecPriority::Backing, + PvfExecKind::Backing, result_tx, ) .await @@ -1647,7 +1647,7 @@ pub(crate) mod tests { pvd.clone(), pov.clone(), Priority::Critical, - PvfExecPriority::Backing, + PvfExecKind::Backing, result_tx_2, ) .await @@ -1671,7 +1671,7 @@ pub(crate) mod tests { pvd.clone(), pov.clone(), Priority::Critical, - PvfExecPriority::Backing, + PvfExecKind::Backing, result_tx_3, ) .await @@ -1730,7 +1730,7 @@ pub(crate) mod tests { pvd.clone(), pov.clone(), Priority::Critical, - PvfExecPriority::Backing, + PvfExecKind::Backing, result_tx, ) .await @@ -1762,7 +1762,7 @@ pub(crate) mod tests { pvd.clone(), pov.clone(), Priority::Critical, - PvfExecPriority::Backing, + PvfExecKind::Backing, result_tx_2, ) .await @@ -1786,7 +1786,7 @@ pub(crate) mod tests { pvd.clone(), pov.clone(), Priority::Critical, - PvfExecPriority::Backing, + PvfExecKind::Backing, result_tx_3, ) .await @@ -1861,7 +1861,7 @@ pub(crate) mod tests { pvd, pov, Priority::Normal, - PvfExecPriority::Backing, + PvfExecKind::Backing, result_tx, ) .await diff --git a/polkadot/node/core/pvf/src/metrics.rs b/polkadot/node/core/pvf/src/metrics.rs index 85f5bc9d7fb25..0dc311ed2f5a9 100644 --- a/polkadot/node/core/pvf/src/metrics.rs +++ b/polkadot/node/core/pvf/src/metrics.rs @@ -18,7 +18,7 @@ use polkadot_node_core_pvf_common::prepare::MemoryStats; use polkadot_node_metrics::metrics::{self, prometheus}; -use polkadot_node_subsystem::messages::PvfExecPriority; +use polkadot_node_subsystem::messages::PvfExecKind; /// Validation host metrics. #[derive(Default, Clone)] @@ -123,7 +123,7 @@ impl Metrics { } /// When preparation pipeline concluded working on an item. - pub(crate) fn on_execute_priority(&self, priority: PvfExecPriority) { + pub(crate) fn on_execute_priority(&self, priority: PvfExecKind) { if let Some(metrics) = &self.0 { metrics.execute_priority_selected.with_label_values(&[priority.as_str()]).inc(); } diff --git a/polkadot/node/core/pvf/src/priority.rs b/polkadot/node/core/pvf/src/priority.rs index 39bb47f4a65c3..7aaeacf362209 100644 --- a/polkadot/node/core/pvf/src/priority.rs +++ b/polkadot/node/core/pvf/src/priority.rs @@ -14,7 +14,7 @@ // You should have received a copy of the GNU General Public License // along with Polkadot. If not, see . -use polkadot_node_subsystem::messages::PvfExecPriority; +use polkadot_node_subsystem::messages::PvfExecKind; /// A priority assigned to preparation of a PVF. #[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord)] @@ -38,13 +38,13 @@ impl Priority { } } -impl From for Priority { - fn from(priority: PvfExecPriority) -> Self { +impl From for Priority { + fn from(priority: PvfExecKind) -> Self { match priority { - PvfExecPriority::Dispute => Priority::Critical, - PvfExecPriority::Approval => Priority::Critical, - PvfExecPriority::BackingSystemParas => Priority::Normal, - PvfExecPriority::Backing => Priority::Normal, + PvfExecKind::Dispute => Priority::Critical, + PvfExecKind::Approval => Priority::Critical, + PvfExecKind::BackingSystemParas => Priority::Normal, + PvfExecKind::Backing => Priority::Normal, } } } diff --git a/polkadot/node/core/pvf/tests/it/main.rs b/polkadot/node/core/pvf/tests/it/main.rs index 5c3a97b9ec049..7b458020d9b6f 100644 --- a/polkadot/node/core/pvf/tests/it/main.rs +++ b/polkadot/node/core/pvf/tests/it/main.rs @@ -25,7 +25,7 @@ use polkadot_node_core_pvf::{ ValidationHost, JOB_TIMEOUT_WALL_CLOCK_FACTOR, }; use polkadot_node_primitives::{PoV, POV_BOMB_LIMIT, VALIDATION_CODE_BOMB_LIMIT}; -use polkadot_node_subsystem::messages::PvfExecPriority; +use polkadot_node_subsystem::messages::PvfExecKind; use polkadot_parachain_primitives::primitives::{BlockData, ValidationResult}; use polkadot_primitives::{ ExecutorParam, ExecutorParams, PersistedValidationData, PvfExecKind, PvfPrepKind, @@ -124,7 +124,7 @@ impl TestHost { Arc::new(pvd), Arc::new(pov), polkadot_node_core_pvf::Priority::Normal, - PvfExecPriority::Backing, + PvfExecKind::Backing, result_tx, ) .await diff --git a/polkadot/node/overseer/examples/minimal-example.rs b/polkadot/node/overseer/examples/minimal-example.rs index 4f8e05d285ce0..26297571465db 100644 --- a/polkadot/node/overseer/examples/minimal-example.rs +++ b/polkadot/node/overseer/examples/minimal-example.rs @@ -24,7 +24,7 @@ use orchestra::async_trait; use std::time::Duration; use polkadot_node_primitives::{BlockData, PoV}; -use polkadot_node_subsystem_types::messages::{CandidateValidationMessage, PvfExecPriority}; +use polkadot_node_subsystem_types::messages::{CandidateValidationMessage, PvfExecKind}; use polkadot_overseer::{ self as overseer, dummy::dummy_overseer_builder, @@ -77,7 +77,7 @@ impl Subsystem1 { candidate_receipt, pov: PoV { block_data: BlockData(Vec::new()) }.into(), executor_params: Default::default(), - exec_kind: PvfExecPriority::Backing, + exec_kind: PvfExecKind::Backing, response_sender: tx, }; ctx.send_message(msg).await; diff --git a/polkadot/node/overseer/src/tests.rs b/polkadot/node/overseer/src/tests.rs index 5ceaeea2edfef..b5efda2fa0cf4 100644 --- a/polkadot/node/overseer/src/tests.rs +++ b/polkadot/node/overseer/src/tests.rs @@ -25,7 +25,7 @@ use polkadot_node_primitives::{ }; use polkadot_node_subsystem_test_helpers::mock::{dummy_unpin_handle, new_leaf}; use polkadot_node_subsystem_types::messages::{ - NetworkBridgeEvent, PvfExecPriority, ReportPeerMessage, RuntimeApiRequest, + NetworkBridgeEvent, PvfExecKind, ReportPeerMessage, RuntimeApiRequest, }; use polkadot_primitives::{ CandidateHash, CandidateReceipt, CollatorPair, Id as ParaId, InvalidDisputeStatementKind, @@ -108,7 +108,7 @@ where candidate_receipt, pov: PoV { block_data: BlockData(Vec::new()) }.into(), executor_params: Default::default(), - exec_kind: PvfExecPriority::Backing, + exec_kind: PvfExecKind::Backing, response_sender: tx, }) .await; @@ -806,7 +806,7 @@ fn test_candidate_validation_msg() -> CandidateValidationMessage { candidate_receipt, pov, executor_params: Default::default(), - exec_kind: PvfExecPriority::Backing, + exec_kind: PvfExecKind::Backing, response_sender, } } diff --git a/polkadot/node/subsystem-types/src/messages.rs b/polkadot/node/subsystem-types/src/messages.rs index 07be9d3793adf..5fb87bdeccd79 100644 --- a/polkadot/node/subsystem-types/src/messages.rs +++ b/polkadot/node/subsystem-types/src/messages.rs @@ -48,9 +48,10 @@ use polkadot_primitives::{ CandidateReceipt, CollatorId, CommittedCandidateReceipt, CoreIndex, CoreState, DisputeState, ExecutorParams, GroupIndex, GroupRotationInfo, Hash, HeadData, Header as BlockHeader, Id as ParaId, InboundDownwardMessage, InboundHrmpMessage, MultiDisputeStatementSet, - NodeFeatures, OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, PvfExecKind, - SessionIndex, SessionInfo, SignedAvailabilityBitfield, SignedAvailabilityBitfields, - ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, ValidatorSignature, + NodeFeatures, OccupiedCoreAssumption, PersistedValidationData, PvfCheckStatement, + PvfExecKind as RuntimePvfExecKind, SessionIndex, SessionInfo, SignedAvailabilityBitfield, + SignedAvailabilityBitfields, ValidationCode, ValidationCodeHash, ValidatorId, ValidatorIndex, + ValidatorSignature, }; use polkadot_statement_table::v2::Misbehavior; use std::{ @@ -161,7 +162,7 @@ pub enum CandidateValidationMessage { /// Session's executor parameters executor_params: ExecutorParams, /// Execution kind, used for timeouts and retries (backing/approvals) - exec_kind: PvfExecPriority, + exec_kind: PvfExecKind, /// The sending side of the response channel response_sender: oneshot::Sender>, }, @@ -186,7 +187,7 @@ pub enum CandidateValidationMessage { /// Session's executor parameters executor_params: ExecutorParams, /// Execution kind, used for timeouts and retries (backing/approvals) - exec_kind: PvfExecPriority, + exec_kind: PvfExecKind, /// The sending side of the response channel response_sender: oneshot::Sender>, }, @@ -210,7 +211,7 @@ pub enum CandidateValidationMessage { /// The order is important, because we iterate through the values and assume it is going from higher /// to lowest priority. #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, EnumIter)] -pub enum PvfExecPriority { +pub enum PvfExecKind { /// For dispute requests Dispute, /// For approval requests @@ -221,7 +222,7 @@ pub enum PvfExecPriority { Backing, } -impl PvfExecPriority { +impl PvfExecKind { /// Converts priority level to &str pub fn as_str(&self) -> &str { match *self { @@ -233,13 +234,13 @@ impl PvfExecPriority { } } -impl From for PvfExecKind { - fn from(exec: PvfExecPriority) -> Self { +impl From for RuntimePvfExecKind { + fn from(exec: PvfExecKind) -> Self { match exec { - PvfExecPriority::Dispute => PvfExecKind::Approval, - PvfExecPriority::Approval => PvfExecKind::Approval, - PvfExecPriority::BackingSystemParas => PvfExecKind::Backing, - PvfExecPriority::Backing => PvfExecKind::Backing, + PvfExecKind::Dispute => RuntimePvfExecKind::Approval, + PvfExecKind::Approval => RuntimePvfExecKind::Approval, + PvfExecKind::BackingSystemParas => RuntimePvfExecKind::Backing, + PvfExecKind::Backing => RuntimePvfExecKind::Backing, } } } From d630b34cef188763c48e4e2afbbd287e6e04d250 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Wed, 9 Oct 2024 16:22:49 +0200 Subject: [PATCH 44/48] Fix renaming --- polkadot/node/core/candidate-validation/src/lib.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/polkadot/node/core/candidate-validation/src/lib.rs b/polkadot/node/core/candidate-validation/src/lib.rs index 40dc04634cb61..cdae4da262017 100644 --- a/polkadot/node/core/candidate-validation/src/lib.rs +++ b/polkadot/node/core/candidate-validation/src/lib.rs @@ -46,8 +46,9 @@ use polkadot_primitives::{ DEFAULT_LENIENT_PREPARATION_TIMEOUT, DEFAULT_PRECHECK_PREPARATION_TIMEOUT, }, AuthorityDiscoveryId, CandidateCommitments, CandidateDescriptor, CandidateEvent, - CandidateReceipt, ExecutorParams, Hash, PersistedValidationData, PvfExecKind, PvfPrepKind, - SessionIndex, ValidationCode, ValidationCodeHash, ValidatorId, + CandidateReceipt, ExecutorParams, Hash, PersistedValidationData, + PvfExecKind as RuntimePvfExecKind, PvfPrepKind, SessionIndex, ValidationCode, + ValidationCodeHash, ValidatorId, }; use sp_application_crypto::{AppCrypto, ByteArray}; use sp_keystore::KeystorePtr; @@ -1035,12 +1036,12 @@ fn pvf_prep_timeout(executor_params: &ExecutorParams, kind: PvfPrepKind) -> Dura /// This should be much longer than the backing execution timeout to ensure that in the /// absence of extremely large disparities between hardware, blocks that pass backing are /// considered executable by approval checkers or dispute participants. -fn pvf_exec_timeout(executor_params: &ExecutorParams, kind: PvfExecKind) -> Duration { +fn pvf_exec_timeout(executor_params: &ExecutorParams, kind: RuntimePvfExecKind) -> Duration { if let Some(timeout) = executor_params.pvf_exec_timeout(kind) { return timeout } match kind { - PvfExecKind::Backing => DEFAULT_BACKING_EXECUTION_TIMEOUT, - PvfExecKind::Approval => DEFAULT_APPROVAL_EXECUTION_TIMEOUT, + RuntimePvfExecKind::Backing => DEFAULT_BACKING_EXECUTION_TIMEOUT, + RuntimePvfExecKind::Approval => DEFAULT_APPROVAL_EXECUTION_TIMEOUT, } } From 6964069f78eadc6b8b123cec1ac79f8b01ad6fbf Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Wed, 9 Oct 2024 16:38:55 +0200 Subject: [PATCH 45/48] Rename execute_priority -> exec_kind --- .../node/core/candidate-validation/src/lib.rs | 18 +++++------ .../core/candidate-validation/src/tests.rs | 6 ++-- polkadot/node/core/pvf/src/execute/queue.rs | 20 ++++-------- polkadot/node/core/pvf/src/host.rs | 31 +++++++------------ polkadot/node/core/pvf/src/metrics.rs | 12 +++---- 5 files changed, 36 insertions(+), 51 deletions(-) diff --git a/polkadot/node/core/candidate-validation/src/lib.rs b/polkadot/node/core/candidate-validation/src/lib.rs index cdae4da262017..e875be9b5df9c 100644 --- a/polkadot/node/core/candidate-validation/src/lib.rs +++ b/polkadot/node/core/candidate-validation/src/lib.rs @@ -787,8 +787,8 @@ trait ValidationBackend { pov: Arc, // The priority for the preparation job. prepare_priority: polkadot_node_core_pvf::Priority, - // The priority for the execution job. - execute_priority: PvfExecKind, + // The kind for the execution job. + exec_kind: PvfExecKind, ) -> Result; /// Tries executing a PVF. Will retry once if an error is encountered that may have @@ -809,8 +809,8 @@ trait ValidationBackend { retry_delay: Duration, // The priority for the preparation job. prepare_priority: polkadot_node_core_pvf::Priority, - // The priority for the execution job. - execute_priority: PvfExecKind, + // The kind for the execution job. + exec_kind: PvfExecKind, ) -> Result { let prep_timeout = pvf_prep_timeout(&executor_params, PvfPrepKind::Prepare); // Construct the PVF a single time, since it is an expensive operation. Cloning it is cheap. @@ -832,7 +832,7 @@ trait ValidationBackend { pvd.clone(), pov.clone(), prepare_priority, - execute_priority, + exec_kind, ) .await; if validation_result.is_ok() { @@ -913,7 +913,7 @@ trait ValidationBackend { pvd.clone(), pov.clone(), prepare_priority, - execute_priority, + exec_kind, ) .await; } @@ -938,12 +938,12 @@ impl ValidationBackend for ValidationHost { pov: Arc, // The priority for the preparation job. prepare_priority: polkadot_node_core_pvf::Priority, - // The priority for the execution job. - execute_priority: PvfExecKind, + // The kind for the execution job. + exec_kind: PvfExecKind, ) -> Result { let (tx, rx) = oneshot::channel(); if let Err(err) = self - .execute_pvf(pvf, exec_timeout, pvd, pov, prepare_priority, execute_priority, tx) + .execute_pvf(pvf, exec_timeout, pvd, pov, prepare_priority, exec_kind, tx) .await { return Err(InternalValidationError::HostCommunication(format!( diff --git a/polkadot/node/core/candidate-validation/src/tests.rs b/polkadot/node/core/candidate-validation/src/tests.rs index 3331a9d2cd619..2f7baf4abb61e 100644 --- a/polkadot/node/core/candidate-validation/src/tests.rs +++ b/polkadot/node/core/candidate-validation/src/tests.rs @@ -442,7 +442,7 @@ impl ValidationBackend for MockValidateCandidateBackend { _pvd: Arc, _pov: Arc, _prepare_priority: polkadot_node_core_pvf::Priority, - _execute_priority: PvfExecKind, + _exec_kind: PvfExecKind, ) -> Result { // This is expected to panic if called more times than expected, indicating an error in the // test. @@ -1025,7 +1025,7 @@ impl ValidationBackend for MockPreCheckBackend { _pvd: Arc, _pov: Arc, _prepare_priority: polkadot_node_core_pvf::Priority, - _execute_priority: PvfExecKind, + _exec_kind: PvfExecKind, ) -> Result { unreachable!() } @@ -1180,7 +1180,7 @@ impl ValidationBackend for MockHeadsUp { _pvd: Arc, _pov: Arc, _prepare_priority: polkadot_node_core_pvf::Priority, - _execute_priority: PvfExecKind, + _exec_kind: PvfExecKind, ) -> Result { unreachable!() } diff --git a/polkadot/node/core/pvf/src/execute/queue.rs b/polkadot/node/core/pvf/src/execute/queue.rs index dda094f2d075e..096cec3501b8a 100644 --- a/polkadot/node/core/pvf/src/execute/queue.rs +++ b/polkadot/node/core/pvf/src/execute/queue.rs @@ -76,7 +76,7 @@ pub struct PendingExecutionRequest { pub pov: Arc, pub executor_params: ExecutorParams, pub result_tx: ResultSender, - pub execute_priority: PvfExecKind, + pub exec_kind: PvfExecKind, } struct ExecuteJob { @@ -284,7 +284,7 @@ impl Queue { } else { spawn_extra_worker(self, job); } - self.metrics.on_execute_priority(priority); + self.metrics.on_execute_kind(priority); self.unscheduled.mark_scheduled(priority); } } @@ -306,14 +306,8 @@ async fn purge_dead(metrics: &Metrics, workers: &mut Workers) { fn handle_to_queue(queue: &mut Queue, to_queue: ToQueue) { let ToQueue::Enqueue { artifact, pending_execution_request } = to_queue; - let PendingExecutionRequest { - exec_timeout, - pvd, - pov, - executor_params, - result_tx, - execute_priority, - } = pending_execution_request; + let PendingExecutionRequest { exec_timeout, pvd, pov, executor_params, result_tx, exec_kind } = + pending_execution_request; gum::debug!( target: LOG_TARGET, validation_code_hash = ?artifact.id.code_hash, @@ -330,7 +324,7 @@ fn handle_to_queue(queue: &mut Queue, to_queue: ToQueue) { result_tx, waiting_since: Instant::now(), }; - queue.unscheduled.add(job, execute_priority); + queue.unscheduled.add(job, exec_kind); queue.try_assign_next_job(None); } @@ -692,9 +686,7 @@ impl Unscheduled { fn new() -> Self { Self { - unscheduled: PvfExecKind::iter() - .map(|priority| (priority, VecDeque::new())) - .collect(), + unscheduled: PvfExecKind::iter().map(|priority| (priority, VecDeque::new())).collect(), counter: PvfExecKind::iter().map(|priority| (priority, 0)).collect(), } } diff --git a/polkadot/node/core/pvf/src/host.rs b/polkadot/node/core/pvf/src/host.rs index 998ac7a4f2d08..37cd6fcbf74aa 100644 --- a/polkadot/node/core/pvf/src/host.rs +++ b/polkadot/node/core/pvf/src/host.rs @@ -114,7 +114,7 @@ impl ValidationHost { pvd: Arc, pov: Arc, priority: Priority, - execute_priority: PvfExecKind, + exec_kind: PvfExecKind, result_tx: ResultSender, ) -> Result<(), String> { self.to_host_tx @@ -124,7 +124,7 @@ impl ValidationHost { pvd, pov, priority, - execute_priority, + exec_kind, result_tx, })) .await @@ -157,7 +157,7 @@ struct ExecutePvfInputs { pvd: Arc, pov: Arc, priority: Priority, - execute_priority: PvfExecKind, + exec_kind: PvfExecKind, result_tx: ResultSender, } @@ -548,8 +548,7 @@ async fn handle_execute_pvf( awaiting_prepare: &mut AwaitingPrepare, inputs: ExecutePvfInputs, ) -> Result<(), Fatal> { - let ExecutePvfInputs { pvf, exec_timeout, pvd, pov, priority, execute_priority, result_tx } = - inputs; + let ExecutePvfInputs { pvf, exec_timeout, pvd, pov, priority, exec_kind, result_tx } = inputs; let artifact_id = ArtifactId::from_pvf_prep_data(&pvf); let executor_params = (*pvf.executor_params()).clone(); @@ -571,7 +570,7 @@ async fn handle_execute_pvf( pvd, pov, executor_params, - execute_priority, + exec_kind, result_tx, }, }, @@ -602,7 +601,7 @@ async fn handle_execute_pvf( pvd, pov, executor_params, - execute_priority, + exec_kind, result_tx, }, ) @@ -618,7 +617,7 @@ async fn handle_execute_pvf( pov, executor_params, result_tx, - execute_priority, + exec_kind, }, ); }, @@ -651,7 +650,7 @@ async fn handle_execute_pvf( pvd, pov, executor_params, - execute_priority, + exec_kind, result_tx, }, ) @@ -677,7 +676,7 @@ async fn handle_execute_pvf( pov, executor_params, result_tx, - execute_priority, + exec_kind, }, ) .await?; @@ -800,14 +799,8 @@ async fn handle_prepare_done( // It's finally time to dispatch all the execution requests that were waiting for this artifact // to be prepared. let pending_requests = awaiting_prepare.take(&artifact_id); - for PendingExecutionRequest { - exec_timeout, - pvd, - pov, - executor_params, - result_tx, - execute_priority, - } in pending_requests + for PendingExecutionRequest { exec_timeout, pvd, pov, executor_params, result_tx, exec_kind } in + pending_requests { if result_tx.is_canceled() { // Preparation could've taken quite a bit of time and the requester may be not @@ -832,7 +825,7 @@ async fn handle_prepare_done( pvd, pov, executor_params, - execute_priority, + exec_kind, result_tx, }, }, diff --git a/polkadot/node/core/pvf/src/metrics.rs b/polkadot/node/core/pvf/src/metrics.rs index 0dc311ed2f5a9..745f2de99e589 100644 --- a/polkadot/node/core/pvf/src/metrics.rs +++ b/polkadot/node/core/pvf/src/metrics.rs @@ -123,9 +123,9 @@ impl Metrics { } /// When preparation pipeline concluded working on an item. - pub(crate) fn on_execute_priority(&self, priority: PvfExecKind) { + pub(crate) fn on_execute_kind(&self, kind: PvfExecKind) { if let Some(metrics) = &self.0 { - metrics.execute_priority_selected.with_label_values(&[priority.as_str()]).inc(); + metrics.exec_kind_selected.with_label_values(&[kind.as_str()]).inc(); } } } @@ -154,7 +154,7 @@ struct MetricsInner { preparation_peak_tracked_allocation: prometheus::Histogram, pov_size: prometheus::HistogramVec, code_size: prometheus::Histogram, - execute_priority_selected: prometheus::CounterVec, + exec_kind_selected: prometheus::CounterVec, } impl metrics::Metrics for Metrics { @@ -378,11 +378,11 @@ impl metrics::Metrics for Metrics { )?, registry, )?, - execute_priority_selected: prometheus::register( + exec_kind_selected: prometheus::register( prometheus::CounterVec::new( prometheus::Opts::new( - "polkadot_pvf_execute_priority_selected", - "The total number of selected execute priorities", + "polkadot_pvf_exec_kind_selected", + "The total number of selected execute kinds", ), &["priority"], )?, From 314aa198e3df70c5cad51497ff114399db72462b Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Wed, 9 Oct 2024 17:10:42 +0200 Subject: [PATCH 46/48] Fix minimal-example --- polkadot/node/overseer/examples/minimal-example.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/polkadot/node/overseer/examples/minimal-example.rs b/polkadot/node/overseer/examples/minimal-example.rs index c6dbdf1e6f667..807e7405ff1b7 100644 --- a/polkadot/node/overseer/examples/minimal-example.rs +++ b/polkadot/node/overseer/examples/minimal-example.rs @@ -31,7 +31,7 @@ use polkadot_overseer::{ gen::{FromOrchestra, SpawnedSubsystem}, HeadSupportsParachains, SubsystemError, }; -use polkadot_primitives::{CandidateReceipt, Hash, PersistedValidationData, PvfExecKind}; +use polkadot_primitives::{CandidateReceipt, Hash, PersistedValidationData}; use polkadot_primitives_test_helpers::{ dummy_candidate_descriptor, dummy_hash, dummy_validation_code, }; From 40dbd8e8f4e4f2886365fe1caa5aba24ffe0fc71 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Wed, 9 Oct 2024 17:56:27 +0200 Subject: [PATCH 47/48] Fix imports --- polkadot/node/core/pvf/tests/it/main.rs | 8 +++++--- polkadot/node/overseer/src/tests.rs | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/polkadot/node/core/pvf/tests/it/main.rs b/polkadot/node/core/pvf/tests/it/main.rs index 7b458020d9b6f..4cbc6fb04a8ef 100644 --- a/polkadot/node/core/pvf/tests/it/main.rs +++ b/polkadot/node/core/pvf/tests/it/main.rs @@ -28,7 +28,8 @@ use polkadot_node_primitives::{PoV, POV_BOMB_LIMIT, VALIDATION_CODE_BOMB_LIMIT}; use polkadot_node_subsystem::messages::PvfExecKind; use polkadot_parachain_primitives::primitives::{BlockData, ValidationResult}; use polkadot_primitives::{ - ExecutorParam, ExecutorParams, PersistedValidationData, PvfExecKind, PvfPrepKind, + ExecutorParam, ExecutorParams, PersistedValidationData, PvfExecKind as RuntimePvfExecKind, + PvfPrepKind, }; use sp_core::H256; @@ -582,8 +583,9 @@ async fn artifact_does_not_reprepare_on_non_meaningful_exec_parameter_change() { let cache_dir = host.cache_dir.path(); let set1 = ExecutorParams::default(); - let set2 = - ExecutorParams::from(&[ExecutorParam::PvfExecTimeout(PvfExecKind::Backing, 2500)][..]); + let set2 = ExecutorParams::from( + &[ExecutorParam::PvfExecTimeout(RuntimePvfExecKind::Backing, 2500)][..], + ); let _stats = host .precheck_pvf(test_parachain_halt::wasm_binary_unwrap(), set1) diff --git a/polkadot/node/overseer/src/tests.rs b/polkadot/node/overseer/src/tests.rs index a0747aa4707f9..46864a482e2a6 100644 --- a/polkadot/node/overseer/src/tests.rs +++ b/polkadot/node/overseer/src/tests.rs @@ -29,7 +29,7 @@ use polkadot_node_subsystem_types::messages::{ }; use polkadot_primitives::{ CandidateHash, CandidateReceipt, CollatorPair, Id as ParaId, InvalidDisputeStatementKind, - PersistedValidationData, PvfExecKind, SessionIndex, ValidDisputeStatementKind, ValidatorIndex, + PersistedValidationData, SessionIndex, ValidDisputeStatementKind, ValidatorIndex, }; use polkadot_primitives_test_helpers::{ dummy_candidate_descriptor, dummy_candidate_receipt, dummy_hash, dummy_validation_code, From 9bec252f97412f9a771f3300ffd96910d7b9f0d8 Mon Sep 17 00:00:00 2001 From: Andrei Eres Date: Wed, 9 Oct 2024 18:35:04 +0200 Subject: [PATCH 48/48] Fix format --- polkadot/node/core/approval-voting/src/lib.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/polkadot/node/core/approval-voting/src/lib.rs b/polkadot/node/core/approval-voting/src/lib.rs index 3a894e3dcee9a..0cb977c58021c 100644 --- a/polkadot/node/core/approval-voting/src/lib.rs +++ b/polkadot/node/core/approval-voting/src/lib.rs @@ -38,8 +38,8 @@ use polkadot_node_subsystem::{ ApprovalVotingMessage, AssignmentCheckError, AssignmentCheckResult, AvailabilityRecoveryMessage, BlockDescription, CandidateValidationMessage, ChainApiMessage, ChainSelectionMessage, CheckedIndirectAssignment, CheckedIndirectSignedApprovalVote, - DisputeCoordinatorMessage, HighestApprovedAncestorBlock, PvfExecKind, - RuntimeApiMessage, RuntimeApiRequest, + DisputeCoordinatorMessage, HighestApprovedAncestorBlock, PvfExecKind, RuntimeApiMessage, + RuntimeApiRequest, }, overseer, FromOrchestra, OverseerSignal, SpawnedSubsystem, SubsystemError, SubsystemResult, SubsystemSender,