From c644d99b6d82fc2e8fc43046bd160f7c01661b9e Mon Sep 17 00:00:00 2001 From: Luke Frisken Date: Sun, 20 Jun 2021 23:17:15 +0400 Subject: [PATCH 01/18] work in progress reset round --- phase1-coordinator/src/coordinator.rs | 76 ++++++++++++++++++- .../src/objects/contribution.rs | 24 ++++++ phase1-coordinator/src/objects/round.rs | 47 +++++++++++- phase1-coordinator/src/storage/disk.rs | 9 +++ phase1-coordinator/src/storage/storage.rs | 68 +++++++++++++++++ 5 files changed, 222 insertions(+), 2 deletions(-) diff --git a/phase1-coordinator/src/coordinator.rs b/phase1-coordinator/src/coordinator.rs index b58a306d..e2736cdd 100644 --- a/phase1-coordinator/src/coordinator.rs +++ b/phase1-coordinator/src/coordinator.rs @@ -8,7 +8,16 @@ use crate::{ coordinator_state::{CoordinatorState, DropParticipant, ParticipantInfo, RoundMetrics}, environment::{Deployment, Environment}, objects::{participant::*, task::TaskInitializationError, ContributionFileSignature, LockedLocators, Round, Task}, - storage::{ContributionLocator, ContributionSignatureLocator, Locator, LocatorPath, Object, Storage, StorageLock}, + storage::{ + ContributionLocator, + ContributionSignatureLocator, + Locator, + LocatorPath, + Object, + Storage, + StorageAction, + StorageLock, + }, }; use setup_utils::calculate_hash; @@ -2457,6 +2466,71 @@ impl Coordinator { pub(super) fn signature(&self) -> Arc> { self.signature.clone() } + + pub fn reset_round(&self, started_at: DateTime) -> Result { + // Fetch the current height of the ceremony. + let round_height = self.current_round_height()?; + info!("Restarting current round {}", round_height); + + let mut round = self.get_round(round_height)?; + { + let mut storage = self.storage.write().map_err(|_| CoordinatorError::StorageLockFailed)?; + // TODO: decide if we need a backup + // if !storage.save_backup(backup_tag) { + // error!( + // "Could not save storage backup for round {} with tag {}", + // round_height, backup_tag + // ); + // return Err(CoordinatorError::StorageUpdateFailed); + // } + + // if let Err(error) = storage.remove(&Locator::RoundHeight) { + // error!("Could not remove round height from storage because: {}", error); + // return Err(CoordinatorError::StorageUpdateFailed); + // } + if let Err(error) = storage.insert(Locator::RoundHeight, Object::RoundHeight(round_height - 1)) { + error!("Could not insert round height to storage because: {}", error); + return Err(CoordinatorError::StorageUpdateFailed); + } + + round.reset() + .into_iter() + .map(StorageAction::Remove) + .map(|action| storage.process(action)) + // this probably causes an allocation, performance could be improved + .collect::>()?; + + // Update the round in storage + storage.update(&Locator::RoundState { round_height }, Object::RoundState(round))?; + } + + // If the round is complete, we also need to clear the next round directory. + // No need to back it up since it's derived from the backed up round. + // TODO: implement this + // if round.is_complete() { + // self.environment.round_directory_reset(round_height + 1); + // } + + // { + // let storage = self.storage.write().map_err(|_| CoordinatorError::StorageLockFailed)?; + // let mut storage_lock = StorageLock::Write(storage); + // // Execute the round initialization as the coordinator. + // // On success, the reset round will have been saved to storage. + // self.run_initialization(&mut storage_lock, started_at)?; + // } + + // Fetch the new round height. + // let new_height = self.current_round_height()?; + + // Check that the new height is the same. + // if new_height != round_height { + // error!("Round height after initialization is {}", new_height); + // return Err(CoordinatorError::RoundHeightMismatch); + // } + + info!("Completed restarting round {}", round_height); + Ok(round_height) + } } #[cfg(any(test, feature = "operator"))] diff --git a/phase1-coordinator/src/objects/contribution.rs b/phase1-coordinator/src/objects/contribution.rs index ae52c839..6f76f42c 100644 --- a/phase1-coordinator/src/objects/contribution.rs +++ b/phase1-coordinator/src/objects/contribution.rs @@ -229,4 +229,28 @@ impl Contribution { self.verified = true; Ok(()) } + + /// Get a list containing all the file locators associated with + /// this contribution. + pub(crate) fn get_locators(&self) -> Vec { + let mut paths: Vec = Vec::new(); + + if let Some(path) = self.get_contributed_location().clone() { + paths.push(path) + } + + if let Some(path) = self.get_contributed_signature_location().clone() { + paths.push(path) + } + + if let Some(path) = self.get_verified_location().clone() { + paths.push(path) + } + + if let Some(path) = self.get_verified_signature_location().clone() { + paths.push(path) + } + + paths + } } diff --git a/phase1-coordinator/src/objects/round.rs b/phase1-coordinator/src/objects/round.rs index 4d48dfbb..8f5cfe92 100644 --- a/phase1-coordinator/src/objects/round.rs +++ b/phase1-coordinator/src/objects/round.rs @@ -2,7 +2,15 @@ use crate::{ coordinator_state::DropParticipant, environment::Environment, objects::{participant::*, Chunk}, - storage::{ContributionLocator, ContributionSignatureLocator, Locator, LocatorPath, Object, StorageLock}, + storage::{ + ContributionLocator, + ContributionSignatureLocator, + Locator, + LocatorPath, + Object, + RemoveAction, + StorageLock, + }, CoordinatorError, }; @@ -1074,6 +1082,43 @@ impl Round { warn!("Modified finished_at timestamp for testing only"); } } + + /// Reset this round back to its initial state before + /// contributions started, and remove any + /// contributions/verifications that have been made since the + /// start of the round. Returns a vector of actions to perform on + /// the [crate::storage::Storage], removing the files associated + /// with the contributions that were removed. + pub(crate) fn reset(&mut self) -> Vec { + self.chunks + .iter_mut() + .flat_map(|chunk| { + let contributions_remove: Vec<(u64, Vec)> = chunk.get_contributions() + .iter() + .filter(|(id, _)| **id != 0) // don't remove initial challenge + .map(|(id, contribution)| { + let actions: Vec = contribution.get_locators() + .into_iter() + .map(|path| RemoveAction::new(path)) + .collect(); + (*id, actions) + }) + .collect(); + + chunk.set_lock_holder_unsafe(None); + + let actions: Vec = contributions_remove + .into_iter() + .flat_map(|(contribution_id, actions)| { + chunk.remove_contribution_unsafe(contribution_id); + actions.into_iter() + }) + .collect(); + + actions.into_iter() + }) + .collect() + } } #[cfg(test)] diff --git a/phase1-coordinator/src/storage/disk.rs b/phase1-coordinator/src/storage/disk.rs index 39c39248..3d2357ad 100644 --- a/phase1-coordinator/src/storage/disk.rs +++ b/phase1-coordinator/src/storage/disk.rs @@ -413,6 +413,15 @@ impl Storage for Disk { trace!("Fetched size of {}", self.to_path(&locator)?); Ok(size) } + + fn process(&mut self, action: StorageAction) -> Result<(), CoordinatorError> { + match action { + StorageAction::Remove(remove_action) => { + let locator = remove_action.try_into_locator(self)?; + self.remove(&locator) + } + } + } } impl StorageLocator for Disk { diff --git a/phase1-coordinator/src/storage/storage.rs b/phase1-coordinator/src/storage/storage.rs index ca29a23c..ac279854 100644 --- a/phase1-coordinator/src/storage/storage.rs +++ b/phase1-coordinator/src/storage/storage.rs @@ -256,6 +256,9 @@ pub trait Storage: Send + Sync + StorageLocator + StorageObject { /// Returns the size of the object stored at the given locator. fn size(&self, locator: &Locator) -> Result; + + /// Process a [StorageAction] which mutates the storage. + fn process(&mut self, action: StorageAction) -> Result<(), CoordinatorError>; } /// The path to a resource defined by a [Locator]. @@ -312,6 +315,71 @@ impl TryFrom<&Path> for LocatorPath { } } +/// An enum containing a [Locator] or [LocatorPath]. +/// +/// **Note:** This can probably be refactored out in the future so +/// that we only use [Locator]. +#[derive(Clone)] +pub enum LocatorOrPath { + Path(LocatorPath), + Locator(Locator), +} + +impl LocatorOrPath { + pub fn try_into_locator(self, storage: &impl Storage) -> Result { + match self { + LocatorOrPath::Path(path) => storage.to_locator(&path), + LocatorOrPath::Locator(locator) => Ok(locator), + } + } +} + +impl From for LocatorOrPath { + fn from(path: LocatorPath) -> Self { + Self::Path(path) + } +} + +impl From for LocatorOrPath { + fn from(locator: Locator) -> Self { + Self::Locator(locator) + } +} + +/// An action to remove an item from [Storage]. +#[derive(Clone)] +pub struct RemoveAction { + locator: LocatorOrPath, +} + +impl RemoveAction { + /// Create a new [RemoveAction] + pub fn new(locator: impl Into) -> Self { + Self { + locator: locator.into(), + } + } + + /// Obtain the location of the item to be removed from [Storage] + /// as a [LocatorOrPath]. + pub fn locator_or_path(&self) -> &LocatorOrPath { + &self.locator + } + + /// Obtain the location of the item to be removed from [Storage] + /// as a [Locator]. + pub fn try_into_locator(self, storage: &impl Storage) -> Result { + self.locator.try_into_locator(storage) + } +} + +/// An action taken to mutate [Storage], which can be processed by +/// [Storage::process()]. +#[non_exhaustive] +pub enum StorageAction { + Remove(RemoveAction), +} + pub trait StorageLocator { /// Returns a locator path corresponding to the given locator. fn to_path(&self, locator: &Locator) -> Result; From 06e67473a9ef55df4d5623815658a88bb41b7016 Mon Sep 17 00:00:00 2001 From: Luke Frisken Date: Tue, 22 Jun 2021 22:59:32 +0400 Subject: [PATCH 02/18] more work in progress round reset implementation --- phase1-coordinator/src/coordinator.rs | 113 +- phase1-coordinator/src/coordinator_state.rs | 86 +- phase1-coordinator/src/objects/round.rs | 34 + phase1-coordinator/src/storage/disk.rs | 2 +- phase1-coordinator/src/storage/storage.rs | 4 +- phase1-coordinator/src/testing/coordinator.rs | 11 + .../resources/test_round_1_partial.json | 1777 +++++++++++++++++ 7 files changed, 1952 insertions(+), 75 deletions(-) create mode 100644 phase1-coordinator/src/testing/resources/test_round_1_partial.json diff --git a/phase1-coordinator/src/coordinator.rs b/phase1-coordinator/src/coordinator.rs index e2736cdd..1ea79b40 100644 --- a/phase1-coordinator/src/coordinator.rs +++ b/phase1-coordinator/src/coordinator.rs @@ -2347,65 +2347,73 @@ impl Coordinator { "Dropping participant from storage with the following information: {:#?}", drop ); - // Fetch the current round from storage. - let mut round = match Self::load_current_round(&storage) { - // Case 1 - This is a typical round of the ceremony. - Ok(round) => round, - // Case 2 - The ceremony has not started or storage has failed. - _ => return Err(CoordinatorError::RoundDoesNotExist), - }; // Check the justification and extract the tasks. - let (tasks, replacement) = match drop { - DropParticipant::BanCurrent(data) => (&data.tasks, &data.replacement), - DropParticipant::DropCurrent(data) => (&data.tasks, &data.replacement), - DropParticipant::Inactive(_) => { + let (tasks, replacement, restart_round) = match drop { + DropParticipant::BanCurrent(data) => (&data.tasks, &data.replacement, data.restart_round), + DropParticipant::DropCurrent(data) => (&data.tasks, &data.replacement, data.restart_round), + DropParticipant::DropQueue(_) => { // Participant is not part of the round, therefore // there is nothing to do. return Ok(vec![]); } }; - warn!("Removing locked chunks and all impacted contributions"); + if restart_round { + self.reset_round()?; - // Remove the lock from the specified chunks. - round.remove_locks_unsafe(&mut storage, &drop)?; - warn!("Removed locked chunks"); + // TODO: should this get locators back? + Ok(Vec::new()) + } else { + // Fetch the current round from storage. + let mut round = match Self::load_current_round(&storage) { + // Case 1 - This is a typical round of the ceremony. + Ok(round) => round, + // Case 2 - The ceremony has not started or storage has failed. + _ => return Err(CoordinatorError::RoundDoesNotExist), + }; - // Remove the contributions from the specified chunks. - round.remove_chunk_contributions_unsafe(&mut storage, &drop)?; - warn!("Removed impacted contributions"); + warn!("Removing locked chunks and all impacted contributions"); - // Assign a replacement contributor to the dropped tasks for the current round. - if let Some(replacement_contributor) = replacement { - round.add_replacement_contributor_unsafe(replacement_contributor.clone())?; - warn!("Added a replacement contributor {}", replacement_contributor); - } + // Remove the lock from the specified chunks. + round.remove_locks_unsafe(&mut storage, &drop)?; + warn!("Removed locked chunks"); - // Convert the tasks into contribution file locators. - let locators = tasks - .par_iter() - .map(|task| { - storage - .to_path(&Locator::ContributionFile(ContributionLocator::new( - round.round_height(), - task.chunk_id(), - task.contribution_id(), - true, - ))) - .unwrap() - }) - .collect(); + // Remove the contributions from the specified chunks. + round.remove_chunk_contributions_unsafe(&mut storage, &drop)?; + warn!("Removed impacted contributions"); - // Save the updated round to storage. - storage.update( - &Locator::RoundState { - round_height: round.round_height(), - }, - Object::RoundState(round), - )?; + // Assign a replacement contributor to the dropped tasks for the current round. + if let Some(replacement_contributor) = replacement { + round.add_replacement_contributor_unsafe(replacement_contributor.clone())?; + warn!("Added a replacement contributor {}", replacement_contributor); + } - Ok(locators) + // Convert the tasks into contribution file locators. + let locators = tasks + .par_iter() + .map(|task| { + storage + .to_path(&Locator::ContributionFile(ContributionLocator::new( + round.round_height(), + task.chunk_id(), + task.contribution_id(), + true, + ))) + .unwrap() + }) + .collect(); + + // Save the updated round to storage. + storage.update( + &Locator::RoundState { + round_height: round.round_height(), + }, + Object::RoundState(round), + )?; + + Ok(locators) + } } #[inline] @@ -2467,7 +2475,8 @@ impl Coordinator { self.signature.clone() } - pub fn reset_round(&self, started_at: DateTime) -> Result { + /// Reset/restart the current round. + pub fn reset_round(&self) -> Result { // Fetch the current height of the ceremony. let round_height = self.current_round_height()?; info!("Restarting current round {}", round_height); @@ -2493,17 +2502,23 @@ impl Coordinator { return Err(CoordinatorError::StorageUpdateFailed); } - round.reset() + if let Some(error) = round + .reset() .into_iter() .map(StorageAction::Remove) .map(|action| storage.process(action)) - // this probably causes an allocation, performance could be improved - .collect::>()?; + .find_map(Result::err) + { + return Err(error); + } // Update the round in storage storage.update(&Locator::RoundState { round_height }, Object::RoundState(round))?; } + let mut state = self.state.write().map_err(|_| CoordinatorError::StateLockFailed)?; + state.reset_round(); + // If the round is complete, we also need to clear the next round directory. // No need to back it up since it's derived from the backed up round. // TODO: implement this diff --git a/phase1-coordinator/src/coordinator_state.rs b/phase1-coordinator/src/coordinator_state.rs index 4985b1bb..c23fa34f 100644 --- a/phase1-coordinator/src/coordinator_state.rs +++ b/phase1-coordinator/src/coordinator_state.rs @@ -860,6 +860,26 @@ pub struct RoundMetrics { next_round_after: Option>, } +impl Default for RoundMetrics { + fn default() -> Self { + Self { + number_of_contributors: 0, + number_of_verifiers: 0, + is_round_aggregated: false, + task_timer: HashMap::new(), + seconds_per_task: HashMap::new(), + contributor_average_per_task: None, + verifier_average_per_task: None, + started_aggregation_at: None, + finished_aggregation_at: None, + estimated_finish_time: None, + estimated_aggregation_time: None, + estimated_wait_time: None, + next_round_after: None, + } + } +} + #[derive(Debug, Clone, Serialize, Deserialize)] pub struct CoordinatorState { /// The parameters and settings of this coordinator. @@ -916,6 +936,28 @@ impl CoordinatorState { } } + pub(super) fn reset_round(&mut self) { + if let Some(round_height) = self.current_round_height { + // Add each current participant back into the queue. + for (participant, participant_info) in self.current_contributors.iter().chain(self.current_verifiers.iter()) + { + self.queue.insert( + participant.clone(), + (participant_info.reliability, Some(participant_info.round_height)), + ); + } + + *self = Self { + queue: self.queue.clone(), + banned: self.banned.clone(), + dropped: self.dropped.clone(), + ..Self::new(self.environment.clone()) + }; + + self.initialize(round_height); + } + } + /// /// Initializes the coordinator state by setting the round height & metrics, and instantiating /// the finished contributors and verifiers map for the given round in the coordinator state. @@ -929,21 +971,7 @@ impl CoordinatorState { // Initialize the metrics for this round. if self.current_metrics.is_none() { - self.current_metrics = Some(RoundMetrics { - number_of_contributors: 0, - number_of_verifiers: 0, - is_round_aggregated: false, - task_timer: HashMap::new(), - seconds_per_task: HashMap::new(), - contributor_average_per_task: None, - verifier_average_per_task: None, - started_aggregation_at: None, - finished_aggregation_at: None, - estimated_finish_time: None, - estimated_aggregation_time: None, - estimated_wait_time: None, - next_round_after: None, - }); + self.current_metrics = Some(Default::default()); } // Initialize the finished contributors map for the current round, if it does not exist. @@ -1921,7 +1949,7 @@ impl CoordinatorState { self.rollback_next_round(); } - return Ok(DropParticipant::Inactive(DropInactiveParticipantData { + return Ok(DropParticipant::DropQueue(DropQueueParticipantData { participant: participant.clone(), })); } @@ -2075,10 +2103,14 @@ impl CoordinatorState { // Add the participant info to the dropped participants. self.dropped.push(dropped_info); - // TODO (howardwu): Add a flag guard to this call, and return None, to support - // the 'drop round' feature in the coordinator. - // Assign the replacement contributor to the dropped tasks. - let replacement_contributor = self.add_replacement_contributor_unsafe(bucket_id, time)?; + let (replacement_contributor, restart_round) = if self.environment.coordinator_contributors().is_empty() + { + // There are no replacement contributors so the only option is to restart the round. + (None, true) + } else { + // Assign the replacement contributor to the dropped tasks. + (Some(self.add_replacement_contributor_unsafe(bucket_id, time)?), false) + }; warn!("Dropped {} from the ceremony", participant); @@ -2087,7 +2119,8 @@ impl CoordinatorState { bucket_id, locked_chunks, tasks, - replacement: Some(replacement_contributor), + replacement: replacement_contributor, + restart_round, })); } Participant::Verifier(_id) => { @@ -2118,12 +2151,17 @@ impl CoordinatorState { warn!("Dropped {} from the ceremony", participant); + // Restart the round because there are no verifiers + // left for this round. + let restart_round = self.current_verifiers.is_empty(); + return Ok(DropParticipant::DropCurrent(DropCurrentParticpantData { participant: participant.clone(), bucket_id, locked_chunks, tasks, replacement: None, + restart_round, })); } } @@ -3162,10 +3200,12 @@ pub(crate) struct DropCurrentParticpantData { /// The participant which will replace the participant being /// dropped. pub replacement: Option, + /// Whether the current round should be restarted. + pub restart_round: bool, } #[derive(Debug)] -pub(crate) struct DropInactiveParticipantData { +pub(crate) struct DropQueueParticipantData { /// The participant being dropped. pub participant: Participant, } @@ -3182,7 +3222,7 @@ pub(crate) enum DropParticipant { DropCurrent(DropCurrentParticpantData), /// Coordinator has decided that a participant in the queue is /// inactive and needs to be removed from the queue. - Inactive(DropInactiveParticipantData), + DropQueue(DropQueueParticipantData), } #[cfg(test)] diff --git a/phase1-coordinator/src/objects/round.rs b/phase1-coordinator/src/objects/round.rs index 8f5cfe92..1c67c9e4 100644 --- a/phase1-coordinator/src/objects/round.rs +++ b/phase1-coordinator/src/objects/round.rs @@ -1090,6 +1090,9 @@ impl Round { /// the [crate::storage::Storage], removing the files associated /// with the contributions that were removed. pub(crate) fn reset(&mut self) -> Vec { + self.contributor_ids.clear(); + self.verifier_ids.clear(); + self.chunks .iter_mut() .flat_map(|chunk| { @@ -1167,6 +1170,37 @@ mod tests { assert_eq!(1, round_1.round_height()); } + #[test] + #[serial] + fn test_reset_partial() { + initialize_test_environment(&TEST_ENVIRONMENT); + + let mut round_1 = test_round_1_partial_json().unwrap(); + println!("{:?}", *TEST_CONTRIBUTOR_ID); + assert!(round_1.is_contributor(&TEST_CONTRIBUTOR_ID_2)); + assert!(round_1.is_contributor(&TEST_CONTRIBUTOR_ID_3)); + assert!(round_1.is_verifier(&TEST_VERIFIER_ID_2)); + assert!(round_1.is_verifier(&TEST_VERIFIER_ID_3)); + + let n_contributions = 89; + let n_verifications = 30; + let n_files = 2 * n_contributions + 2 * n_verifications; + + let actions = round_1.reset(); + assert_eq!(n_files, actions.len()); + + assert_eq!(64, round_1.chunks().len()); + + for chunk in round_1.chunks() { + assert!(!chunk.is_locked()); + assert_eq!(1, chunk.get_contributions().len()); + let contribution = chunk.get_contribution(0).unwrap(); + assert!(contribution.is_verified()); + } + + dbg!(round_1); + } + #[test] #[serial] fn test_is_authorized_contributor() { diff --git a/phase1-coordinator/src/storage/disk.rs b/phase1-coordinator/src/storage/disk.rs index 3d2357ad..34686a8b 100644 --- a/phase1-coordinator/src/storage/disk.rs +++ b/phase1-coordinator/src/storage/disk.rs @@ -31,7 +31,7 @@ use std::{ }; use tracing::{debug, error, trace}; -use super::LocatorPath; +use super::{LocatorPath, StorageAction}; #[derive(Debug)] pub struct Disk { diff --git a/phase1-coordinator/src/storage/storage.rs b/phase1-coordinator/src/storage/storage.rs index ac279854..d6d8eb40 100644 --- a/phase1-coordinator/src/storage/storage.rs +++ b/phase1-coordinator/src/storage/storage.rs @@ -319,7 +319,7 @@ impl TryFrom<&Path> for LocatorPath { /// /// **Note:** This can probably be refactored out in the future so /// that we only use [Locator]. -#[derive(Clone)] +#[derive(Clone, PartialEq)] pub enum LocatorOrPath { Path(LocatorPath), Locator(Locator), @@ -347,7 +347,7 @@ impl From for LocatorOrPath { } /// An action to remove an item from [Storage]. -#[derive(Clone)] +#[derive(Clone, PartialEq)] pub struct RemoveAction { locator: LocatorOrPath, } diff --git a/phase1-coordinator/src/testing/coordinator.rs b/phase1-coordinator/src/testing/coordinator.rs index 74c888af..f6858060 100644 --- a/phase1-coordinator/src/testing/coordinator.rs +++ b/phase1-coordinator/src/testing/coordinator.rs @@ -50,6 +50,10 @@ pub static TEST_VERIFIER_ID: Lazy = Lazy::new(|| test_coordinator_v pub static TEST_VERIFIER_ID_2: Lazy = Lazy::new(|| Participant::Verifier(format!("testing-coordinator-verifier-2"))); +/// Verifier ID 2 for testing purposes only. +pub static TEST_VERIFIER_ID_3: Lazy = + Lazy::new(|| Participant::Verifier(format!("testing-coordinator-verifier-3"))); + /// Contributor IDs for testing purposes only. pub static TEST_CONTRIBUTOR_IDS: Lazy> = Lazy::new(|| vec![Lazy::force(&TEST_CONTRIBUTOR_ID).clone()]); @@ -125,6 +129,13 @@ pub fn test_round_1_initial_json() -> anyhow::Result { ))?) } +/// Loads the reference JSON object with a serialized round for testing purposes only. +pub fn test_round_1_partial_json() -> anyhow::Result { + Ok(serde_json::from_str(include_str!( + "resources/test_round_1_partial.json" + ))?) +} + /// Creates the initial round for testing purposes only. pub fn test_round_0() -> anyhow::Result { // Define test storage. diff --git a/phase1-coordinator/src/testing/resources/test_round_1_partial.json b/phase1-coordinator/src/testing/resources/test_round_1_partial.json new file mode 100644 index 00000000..13aaf351 --- /dev/null +++ b/phase1-coordinator/src/testing/resources/test_round_1_partial.json @@ -0,0 +1,1777 @@ +{ + "version": 1, + "height": 1, + "startedAt": "2021-06-22T17:25:21.109306563Z", + "finishedAt": null, + "contributorIds": [ + "testing-coordinator-contributor-3.contributor", + "testing-coordinator-contributor-2.contributor" + ], + "verifierIds": [ + "testing-coordinator-verifier-3.verifier", + "testing-coordinator-verifier-2.verifier" + ], + "chunks": [ + { + "chunkId": 0, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_0/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_0/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_0/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_0/contribution_1.unverified.signature", + "verifierId": "testing-coordinator-verifier-2.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_0/contribution_1.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_0/contribution_1.verified.signature", + "verified": true + }, + "2": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_0/contribution_2.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_0/contribution_2.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 1, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_1/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_1/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_1/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_1/contribution_1.unverified.signature", + "verifierId": "testing-coordinator-verifier-2.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_1/contribution_1.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_1/contribution_1.verified.signature", + "verified": true + }, + "2": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_1/contribution_2.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_1/contribution_2.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 2, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_2/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_2/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_2/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_2/contribution_1.unverified.signature", + "verifierId": "testing-coordinator-verifier-2.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_2/contribution_1.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_2/contribution_1.verified.signature", + "verified": true + }, + "2": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_2/contribution_2.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_2/contribution_2.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 3, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_3/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_3/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_3/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_3/contribution_1.unverified.signature", + "verifierId": "testing-coordinator-verifier-2.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_3/contribution_1.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_3/contribution_1.verified.signature", + "verified": true + }, + "2": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_3/contribution_2.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_3/contribution_2.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 4, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_4/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_4/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_4/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_4/contribution_1.unverified.signature", + "verifierId": "testing-coordinator-verifier-2.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_4/contribution_1.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_4/contribution_1.verified.signature", + "verified": true + }, + "2": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_4/contribution_2.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_4/contribution_2.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 5, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_5/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_5/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_5/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_5/contribution_1.unverified.signature", + "verifierId": "testing-coordinator-verifier-2.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_5/contribution_1.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_5/contribution_1.verified.signature", + "verified": true + }, + "2": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_5/contribution_2.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_5/contribution_2.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 6, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_6/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_6/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_6/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_6/contribution_1.unverified.signature", + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_6/contribution_1.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_6/contribution_1.verified.signature", + "verified": true + }, + "2": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_6/contribution_2.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_6/contribution_2.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 7, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_7/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_7/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_7/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_7/contribution_1.unverified.signature", + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_7/contribution_1.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_7/contribution_1.verified.signature", + "verified": true + }, + "2": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_7/contribution_2.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_7/contribution_2.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 8, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_8/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_8/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_8/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_8/contribution_1.unverified.signature", + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_8/contribution_1.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_8/contribution_1.verified.signature", + "verified": true + }, + "2": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_8/contribution_2.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_8/contribution_2.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 9, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_9/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_9/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_9/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_9/contribution_1.unverified.signature", + "verifierId": "testing-coordinator-verifier-2.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_9/contribution_1.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_9/contribution_1.verified.signature", + "verified": true + } + } + }, + { + "chunkId": 10, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_10/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_10/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_10/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_10/contribution_1.unverified.signature", + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_10/contribution_1.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_10/contribution_1.verified.signature", + "verified": true + } + } + }, + { + "chunkId": 11, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_11/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_11/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_11/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_11/contribution_1.unverified.signature", + "verifierId": "testing-coordinator-verifier-2.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_11/contribution_1.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_11/contribution_1.verified.signature", + "verified": true + } + } + }, + { + "chunkId": 12, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_12/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_12/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_12/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_12/contribution_1.unverified.signature", + "verifierId": "testing-coordinator-verifier-2.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_12/contribution_1.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_12/contribution_1.verified.signature", + "verified": true + } + } + }, + { + "chunkId": 13, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_13/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_13/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_13/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_13/contribution_1.unverified.signature", + "verifierId": "testing-coordinator-verifier-2.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_13/contribution_1.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_13/contribution_1.verified.signature", + "verified": true + } + } + }, + { + "chunkId": 14, + "lockHolder": "testing-coordinator-verifier-3.verifier", + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_14/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_14/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_14/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_14/contribution_1.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 15, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_15/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_15/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_15/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_15/contribution_1.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 16, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_16/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_16/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_16/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_16/contribution_1.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 17, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_17/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_17/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_17/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_17/contribution_1.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 18, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_18/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_18/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_18/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_18/contribution_1.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 19, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_19/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_19/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_19/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_19/contribution_1.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 20, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_20/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_20/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_20/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_20/contribution_1.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 21, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_21/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_21/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_21/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_21/contribution_1.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 22, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_22/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_22/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_22/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_22/contribution_1.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 23, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_23/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_23/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_23/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_23/contribution_1.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 24, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_24/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_24/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_24/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_24/contribution_1.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 25, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_25/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_25/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_25/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_25/contribution_1.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 26, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_26/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_26/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_26/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_26/contribution_1.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 27, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_27/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_27/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_27/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_27/contribution_1.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 28, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_28/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_28/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_28/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_28/contribution_1.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 29, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_29/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_29/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_29/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_29/contribution_1.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 30, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_30/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_30/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_30/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_30/contribution_1.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 31, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_31/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_31/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_31/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_31/contribution_1.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 32, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_32/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_32/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_32/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_32/contribution_1.unverified.signature", + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_32/contribution_1.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_32/contribution_1.verified.signature", + "verified": true + }, + "2": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_32/contribution_2.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_32/contribution_2.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 33, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_33/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_33/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_33/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_33/contribution_1.unverified.signature", + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_33/contribution_1.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_33/contribution_1.verified.signature", + "verified": true + }, + "2": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_33/contribution_2.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_33/contribution_2.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 34, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_34/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_34/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_34/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_34/contribution_1.unverified.signature", + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_34/contribution_1.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_34/contribution_1.verified.signature", + "verified": true + }, + "2": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_34/contribution_2.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_34/contribution_2.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 35, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_35/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_35/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_35/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_35/contribution_1.unverified.signature", + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_35/contribution_1.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_35/contribution_1.verified.signature", + "verified": true + }, + "2": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_35/contribution_2.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_35/contribution_2.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 36, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_36/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_36/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_36/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_36/contribution_1.unverified.signature", + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_36/contribution_1.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_36/contribution_1.verified.signature", + "verified": true + }, + "2": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_36/contribution_2.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_36/contribution_2.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 37, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_37/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_37/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_37/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_37/contribution_1.unverified.signature", + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_37/contribution_1.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_37/contribution_1.verified.signature", + "verified": true + }, + "2": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_37/contribution_2.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_37/contribution_2.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 38, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_38/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_38/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_38/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_38/contribution_1.unverified.signature", + "verifierId": "testing-coordinator-verifier-2.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_38/contribution_1.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_38/contribution_1.verified.signature", + "verified": true + }, + "2": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_38/contribution_2.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_38/contribution_2.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 39, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_39/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_39/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_39/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_39/contribution_1.unverified.signature", + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_39/contribution_1.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_39/contribution_1.verified.signature", + "verified": true + }, + "2": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_39/contribution_2.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_39/contribution_2.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 40, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_40/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_40/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_40/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_40/contribution_1.unverified.signature", + "verifierId": "testing-coordinator-verifier-2.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_40/contribution_1.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_40/contribution_1.verified.signature", + "verified": true + }, + "2": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_40/contribution_2.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_40/contribution_2.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 41, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_41/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_41/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_41/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_41/contribution_1.unverified.signature", + "verifierId": "testing-coordinator-verifier-2.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_41/contribution_1.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_41/contribution_1.verified.signature", + "verified": true + }, + "2": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_41/contribution_2.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_41/contribution_2.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 42, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_42/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_42/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_42/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_42/contribution_1.unverified.signature", + "verifierId": "testing-coordinator-verifier-2.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_42/contribution_1.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_42/contribution_1.verified.signature", + "verified": true + }, + "2": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_42/contribution_2.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_42/contribution_2.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 43, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_43/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_43/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_43/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_43/contribution_1.unverified.signature", + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_43/contribution_1.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_43/contribution_1.verified.signature", + "verified": true + }, + "2": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_43/contribution_2.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_43/contribution_2.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 44, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_44/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_44/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_44/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_44/contribution_1.unverified.signature", + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_44/contribution_1.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_44/contribution_1.verified.signature", + "verified": true + }, + "2": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_44/contribution_2.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_44/contribution_2.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 45, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_45/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_45/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_45/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_45/contribution_1.unverified.signature", + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_45/contribution_1.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_45/contribution_1.verified.signature", + "verified": true + }, + "2": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_45/contribution_2.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_45/contribution_2.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 46, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_46/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_46/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_46/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_46/contribution_1.unverified.signature", + "verifierId": "testing-coordinator-verifier-2.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_46/contribution_1.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_46/contribution_1.verified.signature", + "verified": true + }, + "2": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_46/contribution_2.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_46/contribution_2.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 47, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_47/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_47/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_47/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_47/contribution_1.unverified.signature", + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_47/contribution_1.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_47/contribution_1.verified.signature", + "verified": true + }, + "2": { + "contributorId": "testing-coordinator-contributor-3.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_47/contribution_2.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_47/contribution_2.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 48, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_48/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_48/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_48/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_48/contribution_1.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 49, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_49/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_49/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_49/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_49/contribution_1.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 50, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_50/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_50/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_50/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_50/contribution_1.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 51, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_51/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_51/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_51/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_51/contribution_1.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 52, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_52/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_52/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_52/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_52/contribution_1.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 53, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_53/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_53/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_53/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_53/contribution_1.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 54, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_54/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_54/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_54/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_54/contribution_1.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 55, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_55/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_55/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_55/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_55/contribution_1.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 56, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_56/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_56/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_56/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_56/contribution_1.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 57, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_57/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_57/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_57/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_57/contribution_1.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 58, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_58/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_58/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_58/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_58/contribution_1.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 59, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_59/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_59/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_59/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_59/contribution_1.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 60, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_60/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_60/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_60/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_60/contribution_1.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 61, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_61/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_61/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_61/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_61/contribution_1.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 62, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_62/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_62/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_62/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_62/contribution_1.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + }, + { + "chunkId": 63, + "lockHolder": null, + "contributions": { + "0": { + "contributorId": null, + "contributedLocation": null, + "contributedSignatureLocation": null, + "verifierId": "testing-coordinator-verifier-3.verifier", + "verifiedLocation": "./transcript/development/round_1/chunk_63/contribution_0.verified", + "verifiedSignatureLocation": "./transcript/development/round_1/chunk_63/contribution_0.verified.signature", + "verified": true + }, + "1": { + "contributorId": "testing-coordinator-contributor-2.contributor", + "contributedLocation": "./transcript/development/round_1/chunk_63/contribution_1.unverified", + "contributedSignatureLocation": "./transcript/development/round_1/chunk_63/contribution_1.unverified.signature", + "verifierId": null, + "verifiedLocation": null, + "verifiedSignatureLocation": null, + "verified": false + } + } + } + ] +} \ No newline at end of file From b08318376d0dc0d9cc950762314b1c53de1148c5 Mon Sep 17 00:00:00 2001 From: Luke Frisken Date: Sat, 26 Jun 2021 22:23:44 +0400 Subject: [PATCH 03/18] CoordinatorState reset round implemented with unit test passing --- phase1-coordinator/src/coordinator.rs | 2 +- phase1-coordinator/src/coordinator_state.rs | 268 +++++++++++++++++++- 2 files changed, 259 insertions(+), 11 deletions(-) diff --git a/phase1-coordinator/src/coordinator.rs b/phase1-coordinator/src/coordinator.rs index 1ea79b40..ecdfcd43 100644 --- a/phase1-coordinator/src/coordinator.rs +++ b/phase1-coordinator/src/coordinator.rs @@ -2517,7 +2517,7 @@ impl Coordinator { } let mut state = self.state.write().map_err(|_| CoordinatorError::StateLockFailed)?; - state.reset_round(); + state.reset_round(&*self.time)?; // If the round is complete, we also need to clear the next round directory. // No need to back it up since it's derived from the backed up round. diff --git a/phase1-coordinator/src/coordinator_state.rs b/phase1-coordinator/src/coordinator_state.rs index c23fa34f..b8d44a11 100644 --- a/phase1-coordinator/src/coordinator_state.rs +++ b/phase1-coordinator/src/coordinator_state.rs @@ -231,11 +231,41 @@ impl ParticipantInfo { self.finished_at.is_some() } + /// Clear all the tasks associated with this participant. + fn clear_tasks(&mut self) { + self.pending_tasks = Default::default(); + self.assigned_tasks = Default::default(); + self.completed_tasks = Default::default(); + self.disposing_tasks = Default::default(); + self.disposed_tasks = Default::default(); + } + + /// Clear the locked chunks. + fn clear_locks(&mut self) { + self.locked_chunks = HashMap::new(); + } + + /// Clear the recorded times `started_at`, `dropped_at` and + /// `finished_at`. + fn clear_round_times(&mut self) { + self.started_at = None; + self.dropped_at = None; + self.finished_at = None; + } + + /// Clear tasks, locks and round times, and start this contributor + /// again, assigning it new tasks. + fn restart(&mut self, tasks: LinkedList, time: &dyn TimeSource) -> Result<(), CoordinatorError> { + self.clear_tasks(); + self.clear_locks(); + self.clear_round_times(); + self.start(tasks, time) + } + /// /// Assigns the participant to the given chunks for the current round, /// and sets the start time as the current time. /// - #[inline] fn start(&mut self, tasks: LinkedList, time: &dyn TimeSource) -> Result<(), CoordinatorError> { trace!("Starting {}", self.id); @@ -936,18 +966,67 @@ impl CoordinatorState { } } - pub(super) fn reset_round(&mut self) { + pub(super) fn reset_round(&mut self, time: &dyn TimeSource) -> Result<(), CoordinatorError> { + let span = tracing::error_span!("reset_round", round = self.current_round_height.unwrap_or(0)); + let _guard = span.enter(); + + tracing::info!("Resetting round."); if let Some(round_height) = self.current_round_height { // Add each current participant back into the queue. - for (participant, participant_info) in self.current_contributors.iter().chain(self.current_verifiers.iter()) - { - self.queue.insert( - participant.clone(), - (participant_info.reliability, Some(participant_info.round_height)), - ); - } + // for (participant, participant_info) in self.current_contributors.iter().chain(self.current_verifiers.iter()) + // { + // self.queue.insert( + // participant.clone(), + // (participant_info.reliability, Some(participant_info.round_height)), + // ); + // } + + // Fetch the number of chunks and bucket size. + let number_of_chunks = self.environment.number_of_chunks() as u64; + + let prev_finished_contributors = self + .finished_contributors + .get(&round_height) + .cloned() + .unwrap_or_else(|| HashMap::new()); + + let number_of_contributors = self.current_contributors.len() + self.finished_contributors.len(); + + let current_contributors = self + .current_contributors + .clone() + .into_iter() + .chain(prev_finished_contributors.into_iter()) + .enumerate() + .map(|(bucket_index, (participant, mut participant_info))| { + let bucket_id = bucket_index as u64; + let tasks = initialize_tasks(bucket_id, number_of_chunks, number_of_contributors as u64)?; + participant_info.restart(tasks, time)?; + Ok((participant, participant_info)) + }) + .collect::, CoordinatorError>>()?; + + let prev_finished_verifiers = self + .finished_verifiers + .get(&round_height) + .cloned() + .unwrap_or_else(|| HashMap::new()); + + let current_verifiers = self + .current_verifiers + .clone() + .into_iter() + .chain(prev_finished_verifiers.into_iter()) + .map(|(participant, mut participant_info)| { + participant_info.restart(LinkedList::new(), time)?; + Ok((participant, participant_info)) + }) + .collect::, CoordinatorError>>()?; *self = Self { + current_contributors, + current_verifiers, + queue: self.queue.clone(), banned: self.banned.clone(), dropped: self.dropped.clone(), @@ -955,7 +1034,10 @@ impl CoordinatorState { }; self.initialize(round_height); + self.update_round_metrics(); } + + Ok(()) } /// @@ -3227,7 +3309,7 @@ pub(crate) enum DropParticipant { #[cfg(test)] mod tests { - use crate::{coordinator_state::*, testing::prelude::*, CoordinatorState, SystemTimeSource}; + use crate::{coordinator_state::*, testing::prelude::*, CoordinatorState, MockTimeSource, SystemTimeSource}; #[test] fn test_new() { @@ -4158,4 +4240,170 @@ mod tests { assert_eq!(0, state.dropped.len()); assert_eq!(0, state.banned.len()); } + + #[test] + fn test_round_reset() { + test_logger(); + + let time = SystemTimeSource::new(); + let environment = TEST_ENVIRONMENT.clone(); + + // Fetch two contributors and two verifiers. + let contributor_1 = TEST_CONTRIBUTOR_ID.clone(); + let contributor_2 = TEST_CONTRIBUTOR_ID_2.clone(); + let verifier_1 = TEST_VERIFIER_ID.clone(); + let verifier_2 = TEST_VERIFIER_ID_2.clone(); + + // Initialize a new coordinator state. + let current_round_height = 5; + let mut state = CoordinatorState::new(environment.clone()); + state.initialize(current_round_height); + state.add_to_queue(contributor_1.clone(), 10).unwrap(); + state.add_to_queue(contributor_2.clone(), 9).unwrap(); + state.add_to_queue(verifier_1.clone(), 10).unwrap(); + state.add_to_queue(verifier_2.clone(), 9).unwrap(); + state.update_queue().unwrap(); + state.aggregating_current_round(&time).unwrap(); + state.aggregated_current_round(&time).unwrap(); + + // Advance the coordinator to the next round. + let next_round_height = current_round_height + 1; + state.precommit_next_round(next_round_height, &time).unwrap(); + state.commit_next_round(); + + let number_of_chunks = environment.number_of_chunks(); + let chunks_3_4: u64 = (number_of_chunks * 3) / 4; + + for _ in 0..chunks_3_4 { + // Contributor 1 + { + // Fetch a pending task for the contributor. + let task = state.fetch_task(&contributor_1, &time).unwrap(); + state.acquired_lock(&contributor_1, task.chunk_id(), &time).unwrap(); + let assigned_verifier = state.completed_task(&contributor_1, task, &time).unwrap(); + // Fetch a pending task for the verifier. + let task = state.fetch_task(&assigned_verifier, &time).unwrap(); + + state.acquired_lock(&assigned_verifier, task.chunk_id(), &time).unwrap(); + state.completed_task(&assigned_verifier, task, &time).unwrap(); + + { + // Update the current round metrics. + state.update_round_metrics(); + + // Update the state of current round contributors. + state.update_current_contributors(&time).unwrap(); + + // Update the state of current round verifiers. + state.update_current_verifiers(&time).unwrap(); + + // Drop disconnected participants from the current round. + state.update_dropped_participants(&time).unwrap(); + + // Ban any participants who meet the coordinator criteria. + state.update_banned_participants().unwrap(); + } + } + + // Contributor 2 + { + // Fetch a pending task for the contributor. + let task = state.fetch_task(&contributor_2, &time).unwrap(); + state.acquired_lock(&contributor_2, task.chunk_id(), &time).unwrap(); + let assigned_verifier = state.completed_task(&contributor_2, task, &time).unwrap(); + // Fetch a pending task for the verifier. + let task = state.fetch_task(&assigned_verifier, &time).unwrap(); + state.acquired_lock(&assigned_verifier, task.chunk_id(), &time).unwrap(); + state.completed_task(&assigned_verifier, task, &time).unwrap(); + + { + // Update the current round metrics. + state.update_round_metrics(); + + // Update the state of current round contributors. + state.update_current_contributors(&time).unwrap(); + + // Update the state of current round verifiers. + state.update_current_verifiers(&time).unwrap(); + + // Drop disconnected participants from the current round. + state.update_dropped_participants(&time).unwrap(); + + // Ban any participants who meet the coordinator criteria. + state.update_banned_participants().unwrap(); + } + } + } + + assert!(!state.is_current_round_finished()); + + let time = MockTimeSource::new(Utc::now()); + + state.reset_round(&time).unwrap(); + + // state.update_queue().unwrap(); + + for _ in 0..number_of_chunks { + // Contributor 1 + { + // Fetch a pending task for the contributor. + let task = state.fetch_task(&contributor_1, &time).unwrap(); + state.acquired_lock(&contributor_1, task.chunk_id(), &time).unwrap(); + let assigned_verifier = state.completed_task(&contributor_1, task, &time).unwrap(); + // Fetch a pending task for the verifier. + let task = state.fetch_task(&assigned_verifier, &time).unwrap(); + + state.acquired_lock(&assigned_verifier, task.chunk_id(), &time).unwrap(); + state.completed_task(&assigned_verifier, task, &time).unwrap(); + + { + // Update the current round metrics. + state.update_round_metrics(); + + // Update the state of current round contributors. + state.update_current_contributors(&time).unwrap(); + + // Update the state of current round verifiers. + state.update_current_verifiers(&time).unwrap(); + + // Drop disconnected participants from the current round. + state.update_dropped_participants(&time).unwrap(); + + // Ban any participants who meet the coordinator criteria. + state.update_banned_participants().unwrap(); + } + } + + // Contributor 2 + { + // Fetch a pending task for the contributor. + let task = state.fetch_task(&contributor_2, &time).unwrap(); + state.acquired_lock(&contributor_2, task.chunk_id(), &time).unwrap(); + let assigned_verifier = state.completed_task(&contributor_2, task, &time).unwrap(); + // Fetch a pending task for the verifier. + let task = state.fetch_task(&assigned_verifier, &time).unwrap(); + state.acquired_lock(&assigned_verifier, task.chunk_id(), &time).unwrap(); + state.completed_task(&assigned_verifier, task, &time).unwrap(); + + { + // Update the current round metrics. + state.update_round_metrics(); + + // Update the state of current round contributors. + state.update_current_contributors(&time).unwrap(); + + // Update the state of current round verifiers. + state.update_current_verifiers(&time).unwrap(); + + // Drop disconnected participants from the current round. + state.update_dropped_participants(&time).unwrap(); + + // Ban any participants who meet the coordinator criteria. + state.update_banned_participants().unwrap(); + } + } + } + + assert!(state.is_current_round_finished()); + } } From be53664b8439cf56648333b158396ba10a1cc9fd Mon Sep 17 00:00:00 2001 From: Luke Frisken Date: Sun, 27 Jun 2021 20:06:47 +0400 Subject: [PATCH 04/18] work in progress round restart changes - also make verifier more robust, upon failures it now resets its tasks --- phase1-coordinator/src/coordinator.rs | 79 ++++++++++----------- phase1-coordinator/src/coordinator_state.rs | 5 +- phase1-coordinator/src/objects/round.rs | 72 +++++++++++++------ phase1-coordinator/src/storage/disk.rs | 1 + phase1-coordinator/src/storage/storage.rs | 7 ++ setup1-verifier/src/verifier.rs | 13 +++- 6 files changed, 112 insertions(+), 65 deletions(-) diff --git a/phase1-coordinator/src/coordinator.rs b/phase1-coordinator/src/coordinator.rs index ecdfcd43..b9b68b20 100644 --- a/phase1-coordinator/src/coordinator.rs +++ b/phase1-coordinator/src/coordinator.rs @@ -8,16 +8,7 @@ use crate::{ coordinator_state::{CoordinatorState, DropParticipant, ParticipantInfo, RoundMetrics}, environment::{Deployment, Environment}, objects::{participant::*, task::TaskInitializationError, ContributionFileSignature, LockedLocators, Round, Task}, - storage::{ - ContributionLocator, - ContributionSignatureLocator, - Locator, - LocatorPath, - Object, - Storage, - StorageAction, - StorageLock, - }, + storage::{ContributionLocator, ContributionSignatureLocator, Locator, LocatorPath, Object, Storage, StorageLock}, }; use setup_utils::calculate_hash; @@ -25,7 +16,7 @@ use chrono::{DateTime, Utc}; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use std::{ fmt, - sync::{Arc, RwLock}, + sync::{Arc, RwLock, RwLockWriteGuard}, }; use tracing::*; @@ -455,7 +446,7 @@ impl Coordinator { // Drop disconnected participants from the current round. for drop in state.update_dropped_participants(self.time.as_ref())? { // Update the round to reflect the coordinator state changes. - self.drop_participant_from_storage(&mut storage, &drop)?; + self.drop_participant_from_storage(&mut state, &mut storage, &drop)?; } state.save(&mut storage)?; @@ -725,7 +716,7 @@ impl Coordinator { let drop = state.drop_participant(participant, self.time.as_ref())?; // Update the round to reflect the coordinator state change. - let locations = self.drop_participant_from_storage(&mut storage, &drop)?; + let locations = self.drop_participant_from_storage(&mut state, &mut storage, &drop)?; // Save the coordinator state in storage. state.save(&mut storage)?; @@ -748,7 +739,7 @@ impl Coordinator { let drop = state.ban_participant(participant, self.time.as_ref())?; // Update the round on disk to reflect the coordinator state change. - let locations = self.drop_participant_from_storage(&mut storage, &drop)?; + let locations = self.drop_participant_from_storage(&mut state, &mut storage, &drop)?; // Save the coordinator state in storage. state.save(&mut storage)?; @@ -1006,7 +997,6 @@ impl Coordinator { /// /// If there are no prior rounds, returns a `CoordinatorError`. /// - #[inline] pub fn get_round(&self, round_height: u64) -> Result { // Acquire the storage lock. let storage = StorageLock::Read(self.storage.read().unwrap()); @@ -2340,7 +2330,8 @@ impl Coordinator { #[inline] fn drop_participant_from_storage( &self, - mut storage: &mut StorageLock, + state: &mut RwLockWriteGuard, + storage: &mut StorageLock, drop: &DropParticipant, ) -> Result, CoordinatorError> { debug!( @@ -2349,9 +2340,9 @@ impl Coordinator { ); // Check the justification and extract the tasks. - let (tasks, replacement, restart_round) = match drop { - DropParticipant::BanCurrent(data) => (&data.tasks, &data.replacement, data.restart_round), - DropParticipant::DropCurrent(data) => (&data.tasks, &data.replacement, data.restart_round), + let drop_data = match drop { + DropParticipant::BanCurrent(data) => data, + DropParticipant::DropCurrent(data) => data, DropParticipant::DropQueue(_) => { // Participant is not part of the round, therefore // there is nothing to do. @@ -2359,8 +2350,8 @@ impl Coordinator { } }; - if restart_round { - self.reset_round()?; + if drop_data.restart_round { + self.reset_round(state, storage, vec![drop_data.participant.clone()])?; // TODO: should this get locators back? Ok(Vec::new()) @@ -2376,21 +2367,22 @@ impl Coordinator { warn!("Removing locked chunks and all impacted contributions"); // Remove the lock from the specified chunks. - round.remove_locks_unsafe(&mut storage, &drop)?; + round.remove_locks_unsafe(storage, &drop)?; warn!("Removed locked chunks"); // Remove the contributions from the specified chunks. - round.remove_chunk_contributions_unsafe(&mut storage, &drop)?; + round.remove_chunk_contributions_unsafe(storage, &drop)?; warn!("Removed impacted contributions"); // Assign a replacement contributor to the dropped tasks for the current round. - if let Some(replacement_contributor) = replacement { + if let Some(replacement_contributor) = &drop_data.replacement { round.add_replacement_contributor_unsafe(replacement_contributor.clone())?; warn!("Added a replacement contributor {}", replacement_contributor); } // Convert the tasks into contribution file locators. - let locators = tasks + let locators = drop_data + .tasks .par_iter() .map(|task| { storage @@ -2475,15 +2467,22 @@ impl Coordinator { self.signature.clone() } - /// Reset/restart the current round. - pub fn reset_round(&self) -> Result { + /// Reset/restart the current round. `remove_participants` is a + /// list of participants to remove from the reset round. + fn reset_round( + &self, + state: &mut RwLockWriteGuard, + storage: &mut StorageLock, + remove_participants: Vec, + ) -> Result { // Fetch the current height of the ceremony. - let round_height = self.current_round_height()?; - info!("Restarting current round {}", round_height); + let round_height = Self::load_current_round_height(storage)?; + let span = tracing::error_span!("reset_round", round = round_height); + let _guard = span.enter(); + info!("Restarting round {}", round_height); - let mut round = self.get_round(round_height)?; + let mut round = Self::load_round(storage, round_height)?; { - let mut storage = self.storage.write().map_err(|_| CoordinatorError::StorageLockFailed)?; // TODO: decide if we need a backup // if !storage.save_backup(backup_tag) { // error!( @@ -2497,27 +2496,25 @@ impl Coordinator { // error!("Could not remove round height from storage because: {}", error); // return Err(CoordinatorError::StorageUpdateFailed); // } - if let Err(error) = storage.insert(Locator::RoundHeight, Object::RoundHeight(round_height - 1)) { - error!("Could not insert round height to storage because: {}", error); - return Err(CoordinatorError::StorageUpdateFailed); - } + // if let Err(error) = storage.insert(Locator::RoundHeight, Object::RoundHeight(round_height - 1)) { + // error!("Could not insert round height to storage because: {}", error); + // return Err(CoordinatorError::StorageUpdateFailed); + // } + tracing::debug!("Resetting round and applying storage changes"); if let Some(error) = round - .reset() + .reset(remove_participants) .into_iter() - .map(StorageAction::Remove) .map(|action| storage.process(action)) .find_map(Result::err) { return Err(error); } - - // Update the round in storage - storage.update(&Locator::RoundState { round_height }, Object::RoundState(round))?; } - let mut state = self.state.write().map_err(|_| CoordinatorError::StateLockFailed)?; + tracing::debug!("Obtaining write lock for state."); state.reset_round(&*self.time)?; + state.save(storage)?; // If the round is complete, we also need to clear the next round directory. // No need to back it up since it's derived from the backed up round. diff --git a/phase1-coordinator/src/coordinator_state.rs b/phase1-coordinator/src/coordinator_state.rs index b8d44a11..e602c084 100644 --- a/phase1-coordinator/src/coordinator_state.rs +++ b/phase1-coordinator/src/coordinator_state.rs @@ -990,7 +990,7 @@ impl CoordinatorState { .cloned() .unwrap_or_else(|| HashMap::new()); - let number_of_contributors = self.current_contributors.len() + self.finished_contributors.len(); + let number_of_contributors = self.current_contributors.len() + prev_finished_contributors.len(); let current_contributors = self .current_contributors @@ -2185,11 +2185,14 @@ impl CoordinatorState { // Add the participant info to the dropped participants. self.dropped.push(dropped_info); + // TODO: also check whether the replacement contributor has already been assigned. let (replacement_contributor, restart_round) = if self.environment.coordinator_contributors().is_empty() { + tracing::info!("No replacement contributors available, the round will be restarted."); // There are no replacement contributors so the only option is to restart the round. (None, true) } else { + tracing::info!("Found a replacement contributor for the dropped contributor."); // Assign the replacement contributor to the dropped tasks. (Some(self.add_replacement_contributor_unsafe(bucket_id, time)?), false) }; diff --git a/phase1-coordinator/src/objects/round.rs b/phase1-coordinator/src/objects/round.rs index 1c67c9e4..82a20094 100644 --- a/phase1-coordinator/src/objects/round.rs +++ b/phase1-coordinator/src/objects/round.rs @@ -9,7 +9,9 @@ use crate::{ LocatorPath, Object, RemoveAction, + StorageAction, StorageLock, + UpdateAction, }, CoordinatorError, }; @@ -1027,14 +1029,6 @@ impl Round { } } - // Remove the given participant from the set of contributor IDs. - self.contributor_ids = self - .contributor_ids - .par_iter() - .cloned() - .filter(|p| p != participant) - .collect(); - Ok(()) } @@ -1087,13 +1081,18 @@ impl Round { /// contributions started, and remove any /// contributions/verifications that have been made since the /// start of the round. Returns a vector of actions to perform on - /// the [crate::storage::Storage], removing the files associated - /// with the contributions that were removed. - pub(crate) fn reset(&mut self) -> Vec { - self.contributor_ids.clear(); - self.verifier_ids.clear(); - - self.chunks + /// the [crate::storage::Storage] to reflect the changes to the + /// round state. `remove_participants` is a list of participants + /// to remove from the round. + pub(crate) fn reset(&mut self, remove_participants: Vec) -> Vec { + // TODO: in some cases it might be necessary to clear these, + // such as when the round only consists of replacement + // contributors. + // self.contributor_ids.clear(); + // self.verifier_ids.clear(); + + let mut actions: Vec = self + .chunks .iter_mut() .flat_map(|chunk| { let contributions_remove: Vec<(u64, Vec)> = chunk.get_contributions() @@ -1110,17 +1109,43 @@ impl Round { chunk.set_lock_holder_unsafe(None); - let actions: Vec = contributions_remove + let actions: Vec = contributions_remove .into_iter() .flat_map(|(contribution_id, actions)| { chunk.remove_contribution_unsafe(contribution_id); actions.into_iter() }) + .map(StorageAction::Remove) .collect(); actions.into_iter() }) - .collect() + .collect(); + + // Remove the requested participants from the set of contributor IDs. + self.contributor_ids = self + .contributor_ids + .par_iter() + .cloned() + .filter(|c| remove_participants.iter().find(|p| p == &c).is_none()) + .collect(); + + // Remove the requested participants from the set of verifier IDs. + self.verifier_ids = self + .verifier_ids + .par_iter() + .cloned() + .filter(|v| remove_participants.iter().find(|p| p == &v).is_none()) + .collect(); + + actions.push(StorageAction::Update(UpdateAction { + locator: Locator::RoundState { + round_height: self.height, + }, + object: Object::RoundState(self.clone()), // PERFORMANCE: clone here is not great for performance + })); + + actions } } @@ -1175,8 +1200,7 @@ mod tests { fn test_reset_partial() { initialize_test_environment(&TEST_ENVIRONMENT); - let mut round_1 = test_round_1_partial_json().unwrap(); - println!("{:?}", *TEST_CONTRIBUTOR_ID); + let mut round_1: Round = test_round_1_partial_json().unwrap(); assert!(round_1.is_contributor(&TEST_CONTRIBUTOR_ID_2)); assert!(round_1.is_contributor(&TEST_CONTRIBUTOR_ID_3)); assert!(round_1.is_verifier(&TEST_VERIFIER_ID_2)); @@ -1185,9 +1209,10 @@ mod tests { let n_contributions = 89; let n_verifications = 30; let n_files = 2 * n_contributions + 2 * n_verifications; + let n_actions = n_files + 1; // include action to update round - let actions = round_1.reset(); - assert_eq!(n_files, actions.len()); + let actions = round_1.reset(vec![TEST_CONTRIBUTOR_ID_2.clone()]); + assert_eq!(n_actions, actions.len()); assert_eq!(64, round_1.chunks().len()); @@ -1198,7 +1223,10 @@ mod tests { assert!(contribution.is_verified()); } - dbg!(round_1); + assert!(!round_1.is_contributor(&*TEST_CONTRIBUTOR_ID_2)); + assert!(round_1.is_contributor(&*TEST_CONTRIBUTOR_ID_3)); + assert!(round_1.is_verifier(&*TEST_VERIFIER_ID_2)); + assert!(round_1.is_verifier(&*TEST_VERIFIER_ID_3)); } #[test] diff --git a/phase1-coordinator/src/storage/disk.rs b/phase1-coordinator/src/storage/disk.rs index 34686a8b..aa9eb96c 100644 --- a/phase1-coordinator/src/storage/disk.rs +++ b/phase1-coordinator/src/storage/disk.rs @@ -420,6 +420,7 @@ impl Storage for Disk { let locator = remove_action.try_into_locator(self)?; self.remove(&locator) } + StorageAction::Update(update_action) => self.update(&update_action.locator, update_action.object), } } } diff --git a/phase1-coordinator/src/storage/storage.rs b/phase1-coordinator/src/storage/storage.rs index d6d8eb40..0c2c0d25 100644 --- a/phase1-coordinator/src/storage/storage.rs +++ b/phase1-coordinator/src/storage/storage.rs @@ -373,11 +373,18 @@ impl RemoveAction { } } +/// An action to update an item in [Storage]. +pub struct UpdateAction { + pub locator: Locator, + pub object: Object, +} + /// An action taken to mutate [Storage], which can be processed by /// [Storage::process()]. #[non_exhaustive] pub enum StorageAction { Remove(RemoveAction), + Update(UpdateAction), } pub trait StorageLocator { diff --git a/setup1-verifier/src/verifier.rs b/setup1-verifier/src/verifier.rs index 751ed44e..2a280145 100644 --- a/setup1-verifier/src/verifier.rs +++ b/setup1-verifier/src/verifier.rs @@ -405,7 +405,18 @@ impl Verifier { loop { // Run the verification operations. if let Err(error) = self.try_verify().await { - error!("{}", error); + error!("Error while verifying {}", error); + + // Clearing and obtaining new tasks + tracing::warn!("Clearing tasks"); + let tasks_lock = self.tasks.lock().await; + let current_tasks = tasks_lock.get_tasks().clone(); + drop(tasks_lock); // lock is required to clear the tasks + for task in current_tasks { + if let Err(error) = self.clear_task(&task).await { + tracing::error!("Error clearing task: {}", error); + } + } } // Sleep for 5 seconds in between iterations. From 2ad879ef31d52d2e58379fc8bab44697a9d561df Mon Sep 17 00:00:00 2001 From: Luke Frisken Date: Sun, 4 Jul 2021 10:42:41 +0400 Subject: [PATCH 05/18] remove the ban variant fo the DropParticipant enum, small refactors in coordinator_state module --- phase1-coordinator/src/coordinator.rs | 1 - phase1-coordinator/src/coordinator_state.rs | 46 ++++++++++++++------- phase1-coordinator/src/objects/round.rs | 2 - 3 files changed, 31 insertions(+), 18 deletions(-) diff --git a/phase1-coordinator/src/coordinator.rs b/phase1-coordinator/src/coordinator.rs index b9b68b20..9ca725b1 100644 --- a/phase1-coordinator/src/coordinator.rs +++ b/phase1-coordinator/src/coordinator.rs @@ -2341,7 +2341,6 @@ impl Coordinator { // Check the justification and extract the tasks. let drop_data = match drop { - DropParticipant::BanCurrent(data) => data, DropParticipant::DropCurrent(data) => data, DropParticipant::DropQueue(_) => { // Participant is not part of the round, therefore diff --git a/phase1-coordinator/src/coordinator_state.rs b/phase1-coordinator/src/coordinator_state.rs index e602c084..777c8ccb 100644 --- a/phase1-coordinator/src/coordinator_state.rs +++ b/phase1-coordinator/src/coordinator_state.rs @@ -1959,13 +1959,21 @@ impl CoordinatorState { // Update the time to trigger the next round. if metrics.next_round_after.is_none() { - metrics.next_round_after = - Some(time.utc_now() + Duration::seconds(self.environment.queue_wait_time() as i64)); + self.set_next_round_after(time); } Ok(()) } + /// Set the `current_metrics` ([RoundMetrics]) `next_round_after` + /// field to the appropriate value specified in the [Environment]. + fn set_next_round_after(&mut self, time: &dyn TimeSource) { + if let Some(metrics) = &mut self.current_metrics { + metrics.next_round_after = + Some(time.utc_now() + Duration::seconds(self.environment.queue_wait_time() as i64)); + } + } + /// /// Rolls back the current round from aggregating in round metrics. /// @@ -2074,7 +2082,7 @@ impl CoordinatorState { }; // Drop the contributor from the current round, and update participant info and coordinator state. - match participant { + let drop = match participant { Participant::Contributor(_id) => { // TODO (howardwu): Optimization only. // ----------------------------------------------------------------------------------- @@ -2185,28 +2193,32 @@ impl CoordinatorState { // Add the participant info to the dropped participants. self.dropped.push(dropped_info); - // TODO: also check whether the replacement contributor has already been assigned. let (replacement_contributor, restart_round) = if self.environment.coordinator_contributors().is_empty() { tracing::info!("No replacement contributors available, the round will be restarted."); // There are no replacement contributors so the only option is to restart the round. (None, true) } else { - tracing::info!("Found a replacement contributor for the dropped contributor."); + // TODO: handle the situation where all replacement contributors are currently engaged. + tracing::info!( + "Found a replacement contributor for the dropped contributor. \ + Assigning replacement contributor to the dropped contributor's tasks." + ); // Assign the replacement contributor to the dropped tasks. (Some(self.add_replacement_contributor_unsafe(bucket_id, time)?), false) }; warn!("Dropped {} from the ceremony", participant); - return Ok(DropParticipant::DropCurrent(DropCurrentParticpantData { + DropParticipant::DropCurrent(DropCurrentParticpantData { participant: participant.clone(), bucket_id, locked_chunks, tasks, replacement: replacement_contributor, restart_round, - })); + ban: false, + }) } Participant::Verifier(_id) => { // Add just the current pending tasks to a pending verifications list. @@ -2240,16 +2252,19 @@ impl CoordinatorState { // left for this round. let restart_round = self.current_verifiers.is_empty(); - return Ok(DropParticipant::DropCurrent(DropCurrentParticpantData { + DropParticipant::DropCurrent(DropCurrentParticpantData { participant: participant.clone(), bucket_id, locked_chunks, tasks, replacement: None, restart_round, - })); + ban: false, + }) } - } + }; + + Ok(drop) } /// @@ -2268,13 +2283,15 @@ impl CoordinatorState { // Drop the participant from the queue, precommit, and current round. match self.drop_participant(participant, time)? { - DropParticipant::DropCurrent(drop_data) => { + DropParticipant::DropCurrent(mut drop_data) => { + drop_data.ban = true; + // Add the participant to the banned list. self.banned.insert(participant.clone()); debug!("{} was banned from the ceremony", participant); - Ok(DropParticipant::BanCurrent(drop_data)) + Ok(DropParticipant::DropCurrent(drop_data)) } _ => Err(CoordinatorError::JustificationInvalid), } @@ -3287,6 +3304,8 @@ pub(crate) struct DropCurrentParticpantData { pub replacement: Option, /// Whether the current round should be restarted. pub restart_round: bool, + /// The dropped participant is to be banned. + pub ban: bool, } #[derive(Debug)] @@ -3299,9 +3318,6 @@ pub(crate) struct DropQueueParticipantData { /// perform the drop. #[derive(Debug)] pub(crate) enum DropParticipant { - /// Coordinator has decided that a participant needs to be banned - /// (for a variety of potential reasons). - BanCurrent(DropCurrentParticpantData), /// Coordinator has decided that a participant needs to be dropped /// (for a variety of potential reasons). DropCurrent(DropCurrentParticpantData), diff --git a/phase1-coordinator/src/objects/round.rs b/phase1-coordinator/src/objects/round.rs index 82a20094..62094253 100644 --- a/phase1-coordinator/src/objects/round.rs +++ b/phase1-coordinator/src/objects/round.rs @@ -813,7 +813,6 @@ impl Round { ) -> Result<(), CoordinatorError> { // Check that the justification is valid for this operation, and fetch the necessary state. let (participant, locked_chunks) = match drop { - DropParticipant::BanCurrent(data) => (&data.participant, &data.locked_chunks), DropParticipant::DropCurrent(data) => (&data.participant, &data.locked_chunks), _ => return Err(CoordinatorError::JustificationInvalid), }; @@ -959,7 +958,6 @@ impl Round { ) -> Result<(), CoordinatorError> { // Check that the justification is valid for this operation, and fetch the necessary state. let (participant, tasks) = match drop { - DropParticipant::BanCurrent(data) => (&data.participant, &data.tasks), DropParticipant::DropCurrent(data) => (&data.participant, &data.tasks), _ => return Err(CoordinatorError::JustificationInvalid), }; From bb360af89ee8e8bdfd24984e6c78f3b3764b97b1 Mon Sep 17 00:00:00 2001 From: Luke Frisken Date: Sun, 4 Jul 2021 11:23:19 +0400 Subject: [PATCH 06/18] move the reset round responsibility into the drop_participant logic --- phase1-coordinator/src/coordinator.rs | 54 ++------- phase1-coordinator/src/coordinator_state.rs | 123 +++++++++++++++++--- 2 files changed, 121 insertions(+), 56 deletions(-) diff --git a/phase1-coordinator/src/coordinator.rs b/phase1-coordinator/src/coordinator.rs index 9ca725b1..7473a826 100644 --- a/phase1-coordinator/src/coordinator.rs +++ b/phase1-coordinator/src/coordinator.rs @@ -16,7 +16,7 @@ use chrono::{DateTime, Utc}; use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use std::{ fmt, - sync::{Arc, RwLock, RwLockWriteGuard}, + sync::{Arc, RwLock}, }; use tracing::*; @@ -446,7 +446,7 @@ impl Coordinator { // Drop disconnected participants from the current round. for drop in state.update_dropped_participants(self.time.as_ref())? { // Update the round to reflect the coordinator state changes. - self.drop_participant_from_storage(&mut state, &mut storage, &drop)?; + self.drop_participant_from_storage(&mut storage, &drop)?; } state.save(&mut storage)?; @@ -716,7 +716,7 @@ impl Coordinator { let drop = state.drop_participant(participant, self.time.as_ref())?; // Update the round to reflect the coordinator state change. - let locations = self.drop_participant_from_storage(&mut state, &mut storage, &drop)?; + let locations = self.drop_participant_from_storage(&mut storage, &drop)?; // Save the coordinator state in storage. state.save(&mut storage)?; @@ -739,7 +739,7 @@ impl Coordinator { let drop = state.ban_participant(participant, self.time.as_ref())?; // Update the round on disk to reflect the coordinator state change. - let locations = self.drop_participant_from_storage(&mut state, &mut storage, &drop)?; + let locations = self.drop_participant_from_storage(&mut storage, &drop)?; // Save the coordinator state in storage. state.save(&mut storage)?; @@ -2330,7 +2330,6 @@ impl Coordinator { #[inline] fn drop_participant_from_storage( &self, - state: &mut RwLockWriteGuard, storage: &mut StorageLock, drop: &DropParticipant, ) -> Result, CoordinatorError> { @@ -2349,8 +2348,8 @@ impl Coordinator { } }; - if drop_data.restart_round { - self.reset_round(state, storage, vec![drop_data.participant.clone()])?; + if drop_data.reset_round { + self.reset_round_storage(storage, vec![drop_data.participant.clone()])?; // TODO: should this get locators back? Ok(Vec::new()) @@ -2466,19 +2465,17 @@ impl Coordinator { self.signature.clone() } - /// Reset/restart the current round. `remove_participants` is a - /// list of participants to remove from the reset round. - fn reset_round( + /// Reset the current round in storage. + fn reset_round_storage( &self, - state: &mut RwLockWriteGuard, storage: &mut StorageLock, remove_participants: Vec, - ) -> Result { + ) -> Result<(), CoordinatorError> { // Fetch the current height of the ceremony. let round_height = Self::load_current_round_height(storage)?; let span = tracing::error_span!("reset_round", round = round_height); let _guard = span.enter(); - info!("Restarting round {}", round_height); + info!("Resetting round {}", round_height); let mut round = Self::load_round(storage, round_height)?; { @@ -2511,36 +2508,9 @@ impl Coordinator { } } - tracing::debug!("Obtaining write lock for state."); - state.reset_round(&*self.time)?; - state.save(storage)?; - - // If the round is complete, we also need to clear the next round directory. - // No need to back it up since it's derived from the backed up round. - // TODO: implement this - // if round.is_complete() { - // self.environment.round_directory_reset(round_height + 1); - // } - - // { - // let storage = self.storage.write().map_err(|_| CoordinatorError::StorageLockFailed)?; - // let mut storage_lock = StorageLock::Write(storage); - // // Execute the round initialization as the coordinator. - // // On success, the reset round will have been saved to storage. - // self.run_initialization(&mut storage_lock, started_at)?; - // } + info!("Finished resetting round {}", round_height); - // Fetch the new round height. - // let new_height = self.current_round_height()?; - - // Check that the new height is the same. - // if new_height != round_height { - // error!("Round height after initialization is {}", new_height); - // return Err(CoordinatorError::RoundHeightMismatch); - // } - - info!("Completed restarting round {}", round_height); - Ok(round_height) + Ok(()) } } diff --git a/phase1-coordinator/src/coordinator_state.rs b/phase1-coordinator/src/coordinator_state.rs index 777c8ccb..76fe0ba7 100644 --- a/phase1-coordinator/src/coordinator_state.rs +++ b/phase1-coordinator/src/coordinator_state.rs @@ -2082,7 +2082,7 @@ impl CoordinatorState { }; // Drop the contributor from the current round, and update participant info and coordinator state. - let drop = match participant { + let drop_data = match participant { Participant::Contributor(_id) => { // TODO (howardwu): Optimization only. // ----------------------------------------------------------------------------------- @@ -2193,8 +2193,7 @@ impl CoordinatorState { // Add the participant info to the dropped participants. self.dropped.push(dropped_info); - let (replacement_contributor, restart_round) = if self.environment.coordinator_contributors().is_empty() - { + let (replacement_contributor, reset_round) = if self.environment.coordinator_contributors().is_empty() { tracing::info!("No replacement contributors available, the round will be restarted."); // There are no replacement contributors so the only option is to restart the round. (None, true) @@ -2210,15 +2209,15 @@ impl CoordinatorState { warn!("Dropped {} from the ceremony", participant); - DropParticipant::DropCurrent(DropCurrentParticpantData { + DropCurrentParticpantData { participant: participant.clone(), bucket_id, locked_chunks, tasks, replacement: replacement_contributor, - restart_round, + reset_round, ban: false, - }) + } } Participant::Verifier(_id) => { // Add just the current pending tasks to a pending verifications list. @@ -2250,21 +2249,25 @@ impl CoordinatorState { // Restart the round because there are no verifiers // left for this round. - let restart_round = self.current_verifiers.is_empty(); + let reset_round = self.current_verifiers.is_empty(); - DropParticipant::DropCurrent(DropCurrentParticpantData { + DropCurrentParticpantData { participant: participant.clone(), bucket_id, locked_chunks, tasks, replacement: None, - restart_round, + reset_round, ban: false, - }) + } } }; - Ok(drop) + if drop_data.reset_round { + self.reset_round(time)?; + } + + Ok(DropParticipant::DropCurrent(drop_data)) } /// @@ -3302,8 +3305,8 @@ pub(crate) struct DropCurrentParticpantData { /// The participant which will replace the participant being /// dropped. pub replacement: Option, - /// Whether the current round should be restarted. - pub restart_round: bool, + /// Whether the current round should be reset/restarted. + pub reset_round: bool, /// The dropped participant is to be banned. pub ban: bool, } @@ -3328,7 +3331,14 @@ pub(crate) enum DropParticipant { #[cfg(test)] mod tests { - use crate::{coordinator_state::*, testing::prelude::*, CoordinatorState, MockTimeSource, SystemTimeSource}; + use crate::{ + coordinator_state::*, + environment::{Parameters, Testing}, + testing::prelude::*, + CoordinatorState, + MockTimeSource, + SystemTimeSource, + }; #[test] fn test_new() { @@ -4425,4 +4435,89 @@ mod tests { assert!(state.is_current_round_finished()); } + + /// Test round reset when all contributors have been dropped. + #[test] + fn test_round_reset_drop_all_contributors() { + test_logger(); + + let time = SystemTimeSource::new(); + + // Set an environment with no replacement contributors. + let environment: Environment = Testing::from(Parameters::Test8Chunks) + .coordinator_contributors(&[]) + .into(); + dbg!(environment.coordinator_contributors()); + + // Fetch two contributors and two verifiers. + let contributor_1 = TEST_CONTRIBUTOR_ID.clone(); + let contributor_2 = TEST_CONTRIBUTOR_ID_2.clone(); + let verifier_1 = TEST_VERIFIER_ID.clone(); + + // Initialize a new coordinator state. + let current_round_height = 5; + let mut state = CoordinatorState::new(environment.clone()); + state.initialize(current_round_height); + state.add_to_queue(contributor_1.clone(), 10).unwrap(); + state.add_to_queue(verifier_1.clone(), 10).unwrap(); + state.update_queue().unwrap(); + state.aggregating_current_round(&time).unwrap(); + state.aggregated_current_round(&time).unwrap(); + + // Advance the coordinator to the next round. + let next_round_height = current_round_height + 1; + state.precommit_next_round(next_round_height, &time).unwrap(); + state.commit_next_round(); + + let number_of_chunks = environment.number_of_chunks(); + let chunks_3_4: u64 = (number_of_chunks * 3) / 4; + + for _ in 0..chunks_3_4 { + // Contributor 1 + { + // Fetch a pending task for the contributor. + let task = state.fetch_task(&contributor_1, &time).unwrap(); + state.acquired_lock(&contributor_1, task.chunk_id(), &time).unwrap(); + let assigned_verifier = state.completed_task(&contributor_1, task, &time).unwrap(); + // Fetch a pending task for the verifier. + let task = state.fetch_task(&assigned_verifier, &time).unwrap(); + + state.acquired_lock(&assigned_verifier, task.chunk_id(), &time).unwrap(); + state.completed_task(&assigned_verifier, task, &time).unwrap(); + + { + // Update the current round metrics. + state.update_round_metrics(); + + // Update the state of current round contributors. + state.update_current_contributors(&time).unwrap(); + + // Update the state of current round verifiers. + state.update_current_verifiers(&time).unwrap(); + + // Drop disconnected participants from the current round. + state.update_dropped_participants(&time).unwrap(); + + // Ban any participants who meet the coordinator criteria. + state.update_banned_participants().unwrap(); + } + } + } + + assert!(!state.is_current_round_finished()); + + let time = MockTimeSource::new(Utc::now()); + + let drop = state.drop_participant(&contributor_1, &time).unwrap(); + + let drop_data = match drop { + DropParticipant::DropCurrent(drop_data) => drop_data, + DropParticipant::DropQueue(_) => panic!("Unexpected drop type: {:?}", drop), + }; + + assert!(drop_data.reset_round); + + dbg!(drop_data); + state.reset_round(&time).unwrap(); + } } From baf80cd1a2e4ad2c262e2814e4cff1d14a8ee35a Mon Sep 17 00:00:00 2001 From: Luke Frisken Date: Sun, 4 Jul 2021 12:19:35 +0400 Subject: [PATCH 07/18] remove return locators from the drop from storage methods because they weren't being used consistently and were only used for testing, shouldn't be part of the public API. Removed ban DropParticipantData field --- phase1-coordinator/src/coordinator.rs | 38 ++++----------- phase1-coordinator/src/coordinator_state.rs | 15 ++---- phase1-coordinator/src/tests.rs | 53 ++++++--------------- 3 files changed, 29 insertions(+), 77 deletions(-) diff --git a/phase1-coordinator/src/coordinator.rs b/phase1-coordinator/src/coordinator.rs index 7473a826..0e330c38 100644 --- a/phase1-coordinator/src/coordinator.rs +++ b/phase1-coordinator/src/coordinator.rs @@ -705,7 +705,7 @@ impl Coordinator { skip(self, participant), fields(participant = %participant) )] - pub fn drop_participant(&self, participant: &Participant) -> Result, CoordinatorError> { + pub fn drop_participant(&self, participant: &Participant) -> Result<(), CoordinatorError> { // Acquire the storage write lock. let mut storage = StorageLock::Write(self.storage.write().unwrap()); @@ -716,19 +716,19 @@ impl Coordinator { let drop = state.drop_participant(participant, self.time.as_ref())?; // Update the round to reflect the coordinator state change. - let locations = self.drop_participant_from_storage(&mut storage, &drop)?; + self.drop_participant_from_storage(&mut storage, &drop)?; // Save the coordinator state in storage. state.save(&mut storage)?; - Ok(locations) + Ok(()) } /// /// Bans the given participant from the ceremony. /// #[inline] - pub fn ban_participant(&self, participant: &Participant) -> Result, CoordinatorError> { + pub fn ban_participant(&self, participant: &Participant) -> Result<(), CoordinatorError> { // Acquire the storage write lock. let mut storage = StorageLock::Write(self.storage.write().unwrap()); @@ -739,12 +739,12 @@ impl Coordinator { let drop = state.ban_participant(participant, self.time.as_ref())?; // Update the round on disk to reflect the coordinator state change. - let locations = self.drop_participant_from_storage(&mut storage, &drop)?; + self.drop_participant_from_storage(&mut storage, &drop)?; // Save the coordinator state in storage. state.save(&mut storage)?; - Ok(locations) + Ok(()) } /// @@ -2332,7 +2332,7 @@ impl Coordinator { &self, storage: &mut StorageLock, drop: &DropParticipant, - ) -> Result, CoordinatorError> { + ) -> Result<(), CoordinatorError> { debug!( "Dropping participant from storage with the following information: {:#?}", drop @@ -2344,15 +2344,12 @@ impl Coordinator { DropParticipant::DropQueue(_) => { // Participant is not part of the round, therefore // there is nothing to do. - return Ok(vec![]); + return Ok(()); } }; if drop_data.reset_round { self.reset_round_storage(storage, vec![drop_data.participant.clone()])?; - - // TODO: should this get locators back? - Ok(Vec::new()) } else { // Fetch the current round from storage. let mut round = match Self::load_current_round(&storage) { @@ -2378,22 +2375,6 @@ impl Coordinator { warn!("Added a replacement contributor {}", replacement_contributor); } - // Convert the tasks into contribution file locators. - let locators = drop_data - .tasks - .par_iter() - .map(|task| { - storage - .to_path(&Locator::ContributionFile(ContributionLocator::new( - round.round_height(), - task.chunk_id(), - task.contribution_id(), - true, - ))) - .unwrap() - }) - .collect(); - // Save the updated round to storage. storage.update( &Locator::RoundState { @@ -2401,9 +2382,8 @@ impl Coordinator { }, Object::RoundState(round), )?; - - Ok(locators) } + Ok(()) } #[inline] diff --git a/phase1-coordinator/src/coordinator_state.rs b/phase1-coordinator/src/coordinator_state.rs index 76fe0ba7..4d9ca1c9 100644 --- a/phase1-coordinator/src/coordinator_state.rs +++ b/phase1-coordinator/src/coordinator_state.rs @@ -2216,7 +2216,6 @@ impl CoordinatorState { tasks, replacement: replacement_contributor, reset_round, - ban: false, } } Participant::Verifier(_id) => { @@ -2258,7 +2257,6 @@ impl CoordinatorState { tasks, replacement: None, reset_round, - ban: false, } } }; @@ -2286,9 +2284,7 @@ impl CoordinatorState { // Drop the participant from the queue, precommit, and current round. match self.drop_participant(participant, time)? { - DropParticipant::DropCurrent(mut drop_data) => { - drop_data.ban = true; - + DropParticipant::DropCurrent(drop_data) => { // Add the participant to the banned list. self.banned.insert(participant.clone()); @@ -3288,6 +3284,10 @@ impl CoordinatorState { } } +pub enum CeremonyDataAction { + ResetRound(u64), +} + /// Data required by the coordinator to drop a participant from the /// ceremony. #[derive(Debug)] @@ -3307,8 +3307,6 @@ pub(crate) struct DropCurrentParticpantData { pub replacement: Option, /// Whether the current round should be reset/restarted. pub reset_round: bool, - /// The dropped participant is to be banned. - pub ban: bool, } #[derive(Debug)] @@ -4516,8 +4514,5 @@ mod tests { }; assert!(drop_data.reset_round); - - dbg!(drop_data); - state.reset_round(&time).unwrap(); } } diff --git a/phase1-coordinator/src/tests.rs b/phase1-coordinator/src/tests.rs index 4e7f518e..db77d357 100644 --- a/phase1-coordinator/src/tests.rs +++ b/phase1-coordinator/src/tests.rs @@ -265,9 +265,7 @@ fn coordinator_drop_contributor_basic() -> anyhow::Result<()> { assert!(!coordinator.is_finished_verifier(&verifier)); // Drop the contributor from the current round. - let locators = coordinator.drop_participant(&contributor1)?; - // Number of files affected by the drop. - assert_eq!(&number_of_chunks - 1, locators.len()); + coordinator.drop_participant(&contributor1)?; assert!(!coordinator.is_queue_contributor(&contributor1)); assert!(!coordinator.is_queue_contributor(&contributor2)); @@ -374,9 +372,7 @@ fn coordinator_drop_contributor_in_between_two_contributors() -> anyhow::Result< assert!(!coordinator.is_finished_verifier(&verifier)); // Drop the contributor from the current round. - let locators = coordinator.drop_participant(&contributor2)?; - // Number of files affected by the drop. - assert_eq!(&number_of_chunks - 1, locators.len()); + coordinator.drop_participant(&contributor2)?; assert!(!coordinator.is_queue_contributor(&contributor1)); assert!(!coordinator.is_queue_contributor(&contributor2)); assert!(!coordinator.is_queue_contributor(&contributor3)); @@ -534,9 +530,7 @@ fn coordinator_drop_contributor_with_contributors_in_pending_tasks() -> anyhow:: } // Drop the contributor from the current round. - let locators = coordinator.drop_participant(&contributor2)?; - // Number of files affected by the drop. - assert_eq!(&number_of_chunks - 2, locators.len()); + coordinator.drop_participant(&contributor2)?; assert!(!coordinator.is_queue_contributor(&contributor1)); assert!(!coordinator.is_queue_contributor(&contributor2)); assert!(!coordinator.is_queue_contributor(&contributor3)); @@ -698,9 +692,7 @@ fn coordinator_drop_contributor_locked_chunks() -> anyhow::Result<()> { coordinator.try_lock(&contributor2)?; // Drop the contributor from the current round. - let locators = coordinator.drop_participant(&contributor2)?; - // Number of files affected by the drop. - assert_eq!(&number_of_chunks - 2, locators.len()); + coordinator.drop_participant(&contributor2)?; assert!(!coordinator.is_queue_contributor(&contributor1)); assert!(!coordinator.is_queue_contributor(&contributor2)); assert!(!coordinator.is_queue_contributor(&contributor3)); @@ -844,9 +836,7 @@ fn coordinator_drop_contributor_removes_contributions() -> anyhow::Result<()> { assert!(!coordinator.is_finished_verifier(&verifier)); // Drop the contributor from the current round. - let locators = coordinator.drop_participant(&contributor1)?; - // Number of files affected by the drop. - assert_eq!(&number_of_chunks - 1, locators.len()); + coordinator.drop_participant(&contributor1)?; assert!(!coordinator.is_queue_contributor(&contributor1)); assert!(!coordinator.is_queue_contributor(&contributor2)); assert!(!coordinator.is_queue_verifier(&verifier)); @@ -1015,9 +1005,7 @@ fn coordinator_drop_contributor_clear_locks() -> anyhow::Result<()> { coordinator.try_lock(&contributor2)?; // Drop the contributor from the current round. - let locators = coordinator.drop_participant(&contributor2)?; - // Number of files affected by the drop. - assert_eq!(&number_of_chunks - 1, locators.len()); + coordinator.drop_participant(&contributor2)?; assert!(!coordinator.is_queue_contributor(&contributor1)); assert!(!coordinator.is_queue_contributor(&contributor2)); assert!(!coordinator.is_queue_contributor(&contributor3)); @@ -1155,9 +1143,7 @@ fn coordinator_drop_contributor_removes_subsequent_contributions() -> anyhow::Re } // Drop one contributor - let locators = coordinator.drop_participant(&contributor1)?; - // Number of files affected by the drop. - assert_eq!(2, locators.len()); + coordinator.drop_participant(&contributor1)?; // Check that the tasks were reassigned properly for (contributor, contributor_info) in coordinator.current_contributors() { @@ -1223,8 +1209,7 @@ fn coordinator_drop_contributor_and_release_locks() { coordinator.try_lock(&contributor_1.participant).unwrap(); // Drop the contributor which have locked the chunk - let locators = coordinator.drop_participant(&contributor_1.participant).unwrap(); - assert_eq!(0, locators.len()); + coordinator.drop_participant(&contributor_1.participant).unwrap(); // Contribute to the round 1 for _ in 0..number_of_chunks { @@ -1512,8 +1497,7 @@ fn coordinator_drop_contributor_and_update_verifier_tasks() { verifier_1.verify(&coordinator).unwrap(); - let locators = coordinator.drop_participant(&contributor_1.participant).unwrap(); - assert_eq!(1, locators.len()); + coordinator.drop_participant(&contributor_1.participant).unwrap(); // Contribute to the round 1 for _ in 0..number_of_chunks { @@ -1628,9 +1612,8 @@ fn coordinator_drop_multiple_contributors() -> anyhow::Result<()> { coordinator.try_lock(&contributor3)?; // Drop the contributor 1 from the current round. - let locators = coordinator.drop_participant(&contributor1)?; + coordinator.drop_participant(&contributor1)?; // Number of files affected by the drop. - assert_eq!(&number_of_chunks - 2, locators.len()); assert!(!coordinator.is_queue_contributor(&contributor1)); assert!(!coordinator.is_queue_contributor(&contributor2)); assert!(!coordinator.is_queue_contributor(&contributor3)); @@ -1645,8 +1628,7 @@ fn coordinator_drop_multiple_contributors() -> anyhow::Result<()> { assert!(!coordinator.is_finished_verifier(&verifier)); // Drop the contributor 2 from the current round. - let locators = coordinator.drop_participant(&contributor2)?; - assert_eq!(&number_of_chunks - 2, locators.len()); + coordinator.drop_participant(&contributor2)?; assert!(!coordinator.is_queue_contributor(&contributor1)); assert!(!coordinator.is_queue_contributor(&contributor2)); assert!(!coordinator.is_queue_contributor(&contributor3)); @@ -1661,8 +1643,7 @@ fn coordinator_drop_multiple_contributors() -> anyhow::Result<()> { assert!(!coordinator.is_finished_verifier(&verifier)); // Drop the contributor 3 from the current round. - let locators = coordinator.drop_participant(&contributor3)?; - assert_eq!(&number_of_chunks - 4, locators.len()); + coordinator.drop_participant(&contributor3)?; assert!(!coordinator.is_queue_contributor(&contributor1)); assert!(!coordinator.is_queue_contributor(&contributor2)); assert!(!coordinator.is_queue_contributor(&contributor3)); @@ -1864,10 +1845,8 @@ fn drop_all_contributors_and_complete_round() -> anyhow::Result<()> { coordinator.update()?; assert_eq!(1, coordinator.current_round_height()?); - let locators = coordinator.drop_participant(&test_contributor_1.participant)?; - assert_eq!(0, locators.len()); - let locators = coordinator.drop_participant(&test_contributor_2.participant)?; - assert_eq!(0, locators.len()); + coordinator.drop_participant(&test_contributor_1.participant)?; + coordinator.drop_participant(&test_contributor_2.participant)?; assert_eq!(false, coordinator.is_queue_contributor(&test_contributor_1.participant)); assert_eq!(false, coordinator.is_queue_contributor(&test_contributor_2.participant)); @@ -1960,9 +1939,7 @@ fn drop_contributor_and_reassign_tasks() -> anyhow::Result<()> { } // Drop the contributor from the current round. - let locators = coordinator.drop_participant(&contributor1)?; - // Number of files affected by the drop. - assert_eq!(number_of_chunks, locators.len()); + coordinator.drop_participant(&contributor1)?; assert_eq!(false, coordinator.is_queue_contributor(&contributor1)); assert_eq!(false, coordinator.is_queue_contributor(&contributor2)); assert_eq!(false, coordinator.is_queue_verifier(&verifier)); From db3bacd87f47698559160867fae94cfbf36d4f05 Mon Sep 17 00:00:00 2001 From: Luke Frisken Date: Sun, 4 Jul 2021 14:13:52 +0400 Subject: [PATCH 08/18] refactor the drop data to be more structured, remove drop data from round --- phase1-coordinator/src/coordinator.rs | 104 +++++++++----- phase1-coordinator/src/coordinator_state.rs | 150 ++++++++++++++------ phase1-coordinator/src/objects/round.rs | 40 ++---- 3 files changed, 196 insertions(+), 98 deletions(-) diff --git a/phase1-coordinator/src/coordinator.rs b/phase1-coordinator/src/coordinator.rs index 0e330c38..29b2eceb 100644 --- a/phase1-coordinator/src/coordinator.rs +++ b/phase1-coordinator/src/coordinator.rs @@ -13,7 +13,6 @@ use crate::{ use setup_utils::calculate_hash; use chrono::{DateTime, Utc}; -use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use std::{ fmt, sync::{Arc, RwLock}, @@ -2348,41 +2347,82 @@ impl Coordinator { } }; - if drop_data.reset_round { - self.reset_round_storage(storage, vec![drop_data.participant.clone()])?; - } else { - // Fetch the current round from storage. - let mut round = match Self::load_current_round(&storage) { - // Case 1 - This is a typical round of the ceremony. - Ok(round) => round, - // Case 2 - The ceremony has not started or storage has failed. - _ => return Err(CoordinatorError::RoundDoesNotExist), - }; - - warn!("Removing locked chunks and all impacted contributions"); - - // Remove the lock from the specified chunks. - round.remove_locks_unsafe(storage, &drop)?; - warn!("Removed locked chunks"); + match &drop_data.storage_action { + crate::coordinator_state::CeremonyStorageAction::ResetRound(_) => { + self.reset_round_storage(storage, vec![drop_data.participant.clone()])?; + } + crate::coordinator_state::CeremonyStorageAction::ReplaceContributor(replace_action) => { + warn!( + "Replacing contributor {} with {}", + replace_action.dropped_contributor, replace_action.replacement_contributor + ); - // Remove the contributions from the specified chunks. - round.remove_chunk_contributions_unsafe(storage, &drop)?; - warn!("Removed impacted contributions"); + // Fetch the current round from storage. + let mut round = Self::load_current_round(&storage)?; + + warn!("Removing locked chunks and all impacted contributions"); + + // Remove the lock from the specified chunks. + round.remove_locks_unsafe( + storage, + &replace_action.dropped_contributor, + &replace_action.locked_chunks, + )?; + warn!("Removed locked chunks"); + + // Remove the contributions from the specified chunks. + round.remove_chunk_contributions_unsafe( + storage, + &replace_action.dropped_contributor, + &replace_action.tasks, + )?; + warn!("Removed impacted contributions"); + + // Assign a replacement contributor to the dropped tasks for the current round. + round.add_replacement_contributor_unsafe(replace_action.replacement_contributor.clone())?; + warn!( + "Added a replacement contributor {}", + replace_action.replacement_contributor + ); - // Assign a replacement contributor to the dropped tasks for the current round. - if let Some(replacement_contributor) = &drop_data.replacement { - round.add_replacement_contributor_unsafe(replacement_contributor.clone())?; - warn!("Added a replacement contributor {}", replacement_contributor); + // Save the updated round to storage. + storage.update( + &Locator::RoundState { + round_height: round.round_height(), + }, + Object::RoundState(round), + )?; + } + crate::coordinator_state::CeremonyStorageAction::RemoveVerifier(remove_action) => { + warn!("Removing verifier {}", remove_action.dropped_verifier); + + // Fetch the current round from storage. + let mut round = Self::load_current_round(&storage)?; + + warn!("Removing locked chunks and all impacted contributions"); + + // Remove the lock from the specified chunks. + round.remove_locks_unsafe(storage, &remove_action.dropped_verifier, &remove_action.locked_chunks)?; + warn!("Removed locked chunks"); + + // Remove the contributions from the specified chunks. + round.remove_chunk_contributions_unsafe( + storage, + &remove_action.dropped_verifier, + &remove_action.tasks, + )?; + warn!("Removed impacted contributions"); + + // Save the updated round to storage. + storage.update( + &Locator::RoundState { + round_height: round.round_height(), + }, + Object::RoundState(round), + )?; } - - // Save the updated round to storage. - storage.update( - &Locator::RoundState { - round_height: round.round_height(), - }, - Object::RoundState(round), - )?; } + Ok(()) } diff --git a/phase1-coordinator/src/coordinator_state.rs b/phase1-coordinator/src/coordinator_state.rs index 4d9ca1c9..ba7299b8 100644 --- a/phase1-coordinator/src/coordinator_state.rs +++ b/phase1-coordinator/src/coordinator_state.rs @@ -966,7 +966,10 @@ impl CoordinatorState { } } - pub(super) fn reset_round(&mut self, time: &dyn TimeSource) -> Result<(), CoordinatorError> { + pub(super) fn reset_round( + &mut self, + time: &dyn TimeSource, + ) -> Result, CoordinatorError> { let span = tracing::error_span!("reset_round", round = self.current_round_height.unwrap_or(0)); let _guard = span.enter(); @@ -1035,9 +1038,14 @@ impl CoordinatorState { self.initialize(round_height); self.update_round_metrics(); - } - Ok(()) + Ok(Some(ResetRoundStorageAction { + round_height: self.current_round_height(), + remove_participants: Vec::new(), + })) + } else { + Ok(None) + } } /// @@ -2082,7 +2090,7 @@ impl CoordinatorState { }; // Drop the contributor from the current round, and update participant info and coordinator state. - let drop_data = match participant { + let storage_action: CeremonyStorageAction = match participant { Participant::Contributor(_id) => { // TODO (howardwu): Optimization only. // ----------------------------------------------------------------------------------- @@ -2193,10 +2201,13 @@ impl CoordinatorState { // Add the participant info to the dropped participants. self.dropped.push(dropped_info); - let (replacement_contributor, reset_round) = if self.environment.coordinator_contributors().is_empty() { + let action = if self.environment.coordinator_contributors().is_empty() { tracing::info!("No replacement contributors available, the round will be restarted."); // There are no replacement contributors so the only option is to restart the round. - (None, true) + CeremonyStorageAction::ResetRound(ResetRoundStorageAction { + round_height: self.current_round_height(), + remove_participants: vec![participant.clone()], + }) } else { // TODO: handle the situation where all replacement contributors are currently engaged. tracing::info!( @@ -2204,19 +2215,20 @@ impl CoordinatorState { Assigning replacement contributor to the dropped contributor's tasks." ); // Assign the replacement contributor to the dropped tasks. - (Some(self.add_replacement_contributor_unsafe(bucket_id, time)?), false) + let replacement_contributor = self.add_replacement_contributor_unsafe(bucket_id, time)?; + + CeremonyStorageAction::ReplaceContributor(ReplaceContributorStorageAction { + dropped_contributor: participant.clone(), + bucket_id, + locked_chunks, + tasks, + replacement_contributor, + }) }; warn!("Dropped {} from the ceremony", participant); - DropCurrentParticpantData { - participant: participant.clone(), - bucket_id, - locked_chunks, - tasks, - replacement: replacement_contributor, - reset_round, - } + action } Participant::Verifier(_id) => { // Add just the current pending tasks to a pending verifications list. @@ -2250,21 +2262,34 @@ impl CoordinatorState { // left for this round. let reset_round = self.current_verifiers.is_empty(); - DropCurrentParticpantData { - participant: participant.clone(), - bucket_id, - locked_chunks, - tasks, - replacement: None, - reset_round, + if reset_round { + CeremonyStorageAction::ResetRound(ResetRoundStorageAction { + round_height: self.current_round_height(), + remove_participants: vec![participant.clone()], + }) + } else { + CeremonyStorageAction::RemoveVerifier(RemoveVerifierStorageAction { + dropped_verifier: participant.clone(), + bucket_id, + locked_chunks, + tasks, + }) } } }; - if drop_data.reset_round { - self.reset_round(time)?; + match storage_action { + CeremonyStorageAction::ResetRound(_) => { + self.reset_round(time)?; + } + _ => {} } + let drop_data = DropCurrentParticpantData { + participant: participant.clone(), + storage_action, + }; + Ok(DropParticipant::DropCurrent(drop_data)) } @@ -3284,29 +3309,74 @@ impl CoordinatorState { } } -pub enum CeremonyDataAction { - ResetRound(u64), +/// Action to update the storage to reflect a round being reset in +/// [CoordinatorState]. +#[derive(Debug)] +pub struct ResetRoundStorageAction { + /// The height of the round to be reset. + round_height: u64, + /// The participants to be removed from the round during the + /// reset. + remove_participants: Vec, } -/// Data required by the coordinator to drop a participant from the -/// ceremony. +/// Action to update the storage to reflect a verifier being +/// removed in [CoordinatorState]. #[derive(Debug)] -pub(crate) struct DropCurrentParticpantData { - /// The participant being dropped. - pub participant: Participant, +pub struct RemoveVerifierStorageAction { + /// The verifier being dropped. + pub dropped_verifier: Participant, /// Determines the starting chunk, and subsequent tasks selected - /// for this participant. See [initialize_tasks] for more + /// for this verifier. See [initialize_tasks] for more /// information about this parameter. pub bucket_id: u64, - /// Chunks currently locked by the participant. + /// Chunks currently locked by the verifier being dropped. pub locked_chunks: Vec, - /// Tasks currently being performed by the participant. + /// Tasks currently being performed by the verifier being + /// dropped. pub tasks: Vec, - /// The participant which will replace the participant being +} + +/// Action to update the storage to reflect a contributor being +/// replaced in [CoordinatorState]. +#[derive(Debug)] +pub struct ReplaceContributorStorageAction { + /// The contributor being dropped. + pub dropped_contributor: Participant, + /// Determines the starting chunk, and subsequent tasks selected + /// for this contributor. See [initialize_tasks] for more + /// information about this parameter. + pub bucket_id: u64, + /// Chunks currently locked by the contributor being dropped. + pub locked_chunks: Vec, + /// Tasks currently being performed by the contributor being dropped. + pub tasks: Vec, + /// The contributor which will replace the contributor being /// dropped. - pub replacement: Option, - /// Whether the current round should be reset/restarted. - pub reset_round: bool, + pub replacement_contributor: Participant, +} + +/// Actions taken to update the round/storage to reflect a change in +/// [CoordinatorState]. +#[derive(Debug)] +pub enum CeremonyStorageAction { + /// See [ResetRoundStorageAction]. + ResetRound(ResetRoundStorageAction), + /// See [ReplaceContributorStorageAction]. + ReplaceContributor(ReplaceContributorStorageAction), + /// See [RemoveVerifierStorageAction]. + RemoveVerifier(RemoveVerifierStorageAction), +} + +/// Data required by the coordinator to drop a participant from the +/// ceremony. +#[derive(Debug)] +pub(crate) struct DropCurrentParticpantData { + /// The participant being dropped. + pub participant: Participant, + /// Action to perform to update the round/storage after the drop + /// to match the current coordinator state. + pub storage_action: CeremonyStorageAction, } #[derive(Debug)] @@ -4449,7 +4519,7 @@ mod tests { // Fetch two contributors and two verifiers. let contributor_1 = TEST_CONTRIBUTOR_ID.clone(); - let contributor_2 = TEST_CONTRIBUTOR_ID_2.clone(); + let _contributor_2 = TEST_CONTRIBUTOR_ID_2.clone(); let verifier_1 = TEST_VERIFIER_ID.clone(); // Initialize a new coordinator state. @@ -4513,6 +4583,6 @@ mod tests { DropParticipant::DropQueue(_) => panic!("Unexpected drop type: {:?}", drop), }; - assert!(drop_data.reset_round); + assert!(matches!(drop_data.storage_action, CeremonyStorageAction::ResetRound(_))); } } diff --git a/phase1-coordinator/src/objects/round.rs b/phase1-coordinator/src/objects/round.rs index 62094253..a81f75f6 100644 --- a/phase1-coordinator/src/objects/round.rs +++ b/phase1-coordinator/src/objects/round.rs @@ -1,5 +1,4 @@ use crate::{ - coordinator_state::DropParticipant, environment::Environment, objects::{participant::*, Chunk}, storage::{ @@ -24,6 +23,8 @@ use serde_diff::SerdeDiff; use std::{collections::HashSet, hash::Hash}; use tracing::{debug, error, trace, warn}; +use super::Task; + /// A helper function used to check that each list of participants is unique. fn has_unique_elements(iter: T) -> bool where @@ -796,10 +797,7 @@ impl Round { /// /// Removes the locks for the current round from the given chunk IDs. /// - /// If the given justification is not valid for this operation, - /// this function will return a `CoordinatorError`. - /// - /// If the given chunk IDs in the justification are not currently locked, + /// If the given chunk IDs are not currently locked, /// this function will return a `CoordinatorError`. /// /// If the given participant is not the current lock holder of the given chunk IDs, @@ -809,14 +807,9 @@ impl Round { pub(crate) fn remove_locks_unsafe( &mut self, storage: &mut StorageLock, - drop: &DropParticipant, + participant: &Participant, + locked_chunks: &[u64], ) -> Result<(), CoordinatorError> { - // Check that the justification is valid for this operation, and fetch the necessary state. - let (participant, locked_chunks) = match drop { - DropParticipant::DropCurrent(data) => (&data.participant, &data.locked_chunks), - _ => return Err(CoordinatorError::JustificationInvalid), - }; - // Sanity check that the participant holds the lock for each specified chunk. let locked_chunks: Vec<_> = locked_chunks .par_iter() @@ -937,31 +930,26 @@ impl Round { /// Removes the contributions from the current round from the /// given (chunk ID, contribution ID) tasks. /// - /// If the given justification is not valid for this operation, - /// this function will return a `CoordinatorError`. /// - /// If the given (chunk ID, contribution ID) tasks in the justification - /// are not currently locked, this function will return a `CoordinatorError`. + /// If the given (chunk ID, contribution ID) tasks are not + /// currently locked, this function will return a + /// `CoordinatorError`. /// - /// If the given participant is not the current lock holder of the given chunk IDs, - /// this function will return a `CoordinatorError`. + /// If the given participant is not the current lock holder of the + /// given chunk IDs, this function will return a + /// `CoordinatorError`. /// #[tracing::instrument( level = "error", - skip(self, storage, drop), + skip(self, storage, tasks), fields(round = self.round_height()) )] pub(crate) fn remove_chunk_contributions_unsafe( &mut self, storage: &mut StorageLock, - drop: &DropParticipant, + participant: &Participant, + tasks: &[Task], ) -> Result<(), CoordinatorError> { - // Check that the justification is valid for this operation, and fetch the necessary state. - let (participant, tasks) = match drop { - DropParticipant::DropCurrent(data) => (&data.participant, &data.tasks), - _ => return Err(CoordinatorError::JustificationInvalid), - }; - // Check if the participant is a verifier. As verifications are not dependent // on each other, no further update is necessary in the round state. if participant.is_verifier() { From 70be9d1a6bfe1512e7663764ec6a617f4fea517b Mon Sep 17 00:00:00 2001 From: Luke Frisken Date: Sun, 4 Jul 2021 16:33:16 +0400 Subject: [PATCH 09/18] change to ResetCurrentRound action --- phase1-coordinator/src/coordinator.rs | 2 +- phase1-coordinator/src/coordinator_state.rs | 26 ++++++++++----------- 2 files changed, 13 insertions(+), 15 deletions(-) diff --git a/phase1-coordinator/src/coordinator.rs b/phase1-coordinator/src/coordinator.rs index 29b2eceb..73bff127 100644 --- a/phase1-coordinator/src/coordinator.rs +++ b/phase1-coordinator/src/coordinator.rs @@ -2348,7 +2348,7 @@ impl Coordinator { }; match &drop_data.storage_action { - crate::coordinator_state::CeremonyStorageAction::ResetRound(_) => { + crate::coordinator_state::CeremonyStorageAction::ResetCurrentRound(_) => { self.reset_round_storage(storage, vec![drop_data.participant.clone()])?; } crate::coordinator_state::CeremonyStorageAction::ReplaceContributor(replace_action) => { diff --git a/phase1-coordinator/src/coordinator_state.rs b/phase1-coordinator/src/coordinator_state.rs index ba7299b8..8c706fda 100644 --- a/phase1-coordinator/src/coordinator_state.rs +++ b/phase1-coordinator/src/coordinator_state.rs @@ -969,7 +969,7 @@ impl CoordinatorState { pub(super) fn reset_round( &mut self, time: &dyn TimeSource, - ) -> Result, CoordinatorError> { + ) -> Result, CoordinatorError> { let span = tracing::error_span!("reset_round", round = self.current_round_height.unwrap_or(0)); let _guard = span.enter(); @@ -1039,8 +1039,7 @@ impl CoordinatorState { self.initialize(round_height); self.update_round_metrics(); - Ok(Some(ResetRoundStorageAction { - round_height: self.current_round_height(), + Ok(Some(ResetCurrentRoundStorageAction { remove_participants: Vec::new(), })) } else { @@ -2204,8 +2203,7 @@ impl CoordinatorState { let action = if self.environment.coordinator_contributors().is_empty() { tracing::info!("No replacement contributors available, the round will be restarted."); // There are no replacement contributors so the only option is to restart the round. - CeremonyStorageAction::ResetRound(ResetRoundStorageAction { - round_height: self.current_round_height(), + CeremonyStorageAction::ResetCurrentRound(ResetCurrentRoundStorageAction { remove_participants: vec![participant.clone()], }) } else { @@ -2263,8 +2261,7 @@ impl CoordinatorState { let reset_round = self.current_verifiers.is_empty(); if reset_round { - CeremonyStorageAction::ResetRound(ResetRoundStorageAction { - round_height: self.current_round_height(), + CeremonyStorageAction::ResetCurrentRound(ResetCurrentRoundStorageAction { remove_participants: vec![participant.clone()], }) } else { @@ -2279,7 +2276,7 @@ impl CoordinatorState { }; match storage_action { - CeremonyStorageAction::ResetRound(_) => { + CeremonyStorageAction::ResetCurrentRound(_) => { self.reset_round(time)?; } _ => {} @@ -3312,9 +3309,7 @@ impl CoordinatorState { /// Action to update the storage to reflect a round being reset in /// [CoordinatorState]. #[derive(Debug)] -pub struct ResetRoundStorageAction { - /// The height of the round to be reset. - round_height: u64, +pub struct ResetCurrentRoundStorageAction { /// The participants to be removed from the round during the /// reset. remove_participants: Vec, @@ -3360,8 +3355,8 @@ pub struct ReplaceContributorStorageAction { /// [CoordinatorState]. #[derive(Debug)] pub enum CeremonyStorageAction { - /// See [ResetRoundStorageAction]. - ResetRound(ResetRoundStorageAction), + /// See [ResetCurrentRoundStorageAction]. + ResetCurrentRound(ResetCurrentRoundStorageAction), /// See [ReplaceContributorStorageAction]. ReplaceContributor(ReplaceContributorStorageAction), /// See [RemoveVerifierStorageAction]. @@ -4583,6 +4578,9 @@ mod tests { DropParticipant::DropQueue(_) => panic!("Unexpected drop type: {:?}", drop), }; - assert!(matches!(drop_data.storage_action, CeremonyStorageAction::ResetRound(_))); + assert!(matches!( + drop_data.storage_action, + CeremonyStorageAction::ResetCurrentRound(_) + )); } } From df35eafb0308f9cd6b35e751cc173dd6d7b2e17e Mon Sep 17 00:00:00 2001 From: Luke Frisken Date: Sun, 4 Jul 2021 16:49:32 +0400 Subject: [PATCH 10/18] reset round now returns an error if there is no current round --- phase1-coordinator/src/coordinator_state.rs | 130 +++++++++----------- 1 file changed, 61 insertions(+), 69 deletions(-) diff --git a/phase1-coordinator/src/coordinator_state.rs b/phase1-coordinator/src/coordinator_state.rs index 8c706fda..35ebc5f7 100644 --- a/phase1-coordinator/src/coordinator_state.rs +++ b/phase1-coordinator/src/coordinator_state.rs @@ -255,7 +255,7 @@ impl ParticipantInfo { /// Clear tasks, locks and round times, and start this contributor /// again, assigning it new tasks. - fn restart(&mut self, tasks: LinkedList, time: &dyn TimeSource) -> Result<(), CoordinatorError> { + fn restart_tasks(&mut self, tasks: LinkedList, time: &dyn TimeSource) -> Result<(), CoordinatorError> { self.clear_tasks(); self.clear_locks(); self.clear_round_times(); @@ -966,85 +966,77 @@ impl CoordinatorState { } } - pub(super) fn reset_round( + /// Reset the progress of the current round, back to how it was in + /// its initialized state, however this does maintain the drop + /// status of participants. + pub fn reset_current_round( &mut self, time: &dyn TimeSource, - ) -> Result, CoordinatorError> { + ) -> Result { let span = tracing::error_span!("reset_round", round = self.current_round_height.unwrap_or(0)); let _guard = span.enter(); - tracing::info!("Resetting round."); - if let Some(round_height) = self.current_round_height { - // Add each current participant back into the queue. - // for (participant, participant_info) in self.current_contributors.iter().chain(self.current_verifiers.iter()) - // { - // self.queue.insert( - // participant.clone(), - // (participant_info.reliability, Some(participant_info.round_height)), - // ); - // } + let round_height = self.current_round_height.ok_or(CoordinatorError::RoundDoesNotExist)?; + tracing::info!("Resetting round {}.", round_height); - // Fetch the number of chunks and bucket size. - let number_of_chunks = self.environment.number_of_chunks() as u64; + // Fetch the number of chunks and bucket size. + let number_of_chunks = self.environment.number_of_chunks() as u64; - let prev_finished_contributors = self - .finished_contributors - .get(&round_height) - .cloned() - .unwrap_or_else(|| HashMap::new()); + let prev_finished_contributors = self + .finished_contributors + .get(&round_height) + .cloned() + .unwrap_or_else(|| HashMap::new()); - let number_of_contributors = self.current_contributors.len() + prev_finished_contributors.len(); + let number_of_contributors = self.current_contributors.len() + prev_finished_contributors.len(); - let current_contributors = self - .current_contributors - .clone() - .into_iter() - .chain(prev_finished_contributors.into_iter()) - .enumerate() - .map(|(bucket_index, (participant, mut participant_info))| { - let bucket_id = bucket_index as u64; - let tasks = initialize_tasks(bucket_id, number_of_chunks, number_of_contributors as u64)?; - participant_info.restart(tasks, time)?; - Ok((participant, participant_info)) - }) - .collect::, CoordinatorError>>()?; + let current_contributors = self + .current_contributors + .clone() + .into_iter() + .chain(prev_finished_contributors.into_iter()) + .enumerate() + .map(|(bucket_index, (participant, mut participant_info))| { + let bucket_id = bucket_index as u64; + let tasks = initialize_tasks(bucket_id, number_of_chunks, number_of_contributors as u64)?; + participant_info.restart_tasks(tasks, time)?; + Ok((participant, participant_info)) + }) + .collect::, CoordinatorError>>()?; - let prev_finished_verifiers = self - .finished_verifiers - .get(&round_height) - .cloned() - .unwrap_or_else(|| HashMap::new()); + let prev_finished_verifiers = self + .finished_verifiers + .get(&round_height) + .cloned() + .unwrap_or_else(|| HashMap::new()); - let current_verifiers = self - .current_verifiers - .clone() - .into_iter() - .chain(prev_finished_verifiers.into_iter()) - .map(|(participant, mut participant_info)| { - participant_info.restart(LinkedList::new(), time)?; - Ok((participant, participant_info)) - }) - .collect::, CoordinatorError>>()?; + let current_verifiers = self + .current_verifiers + .clone() + .into_iter() + .chain(prev_finished_verifiers.into_iter()) + .map(|(participant, mut participant_info)| { + participant_info.restart_tasks(LinkedList::new(), time)?; + Ok((participant, participant_info)) + }) + .collect::, CoordinatorError>>()?; - *self = Self { - current_contributors, - current_verifiers, + *self = Self { + current_contributors, + current_verifiers, - queue: self.queue.clone(), - banned: self.banned.clone(), - dropped: self.dropped.clone(), - ..Self::new(self.environment.clone()) - }; + queue: self.queue.clone(), + banned: self.banned.clone(), + dropped: self.dropped.clone(), + ..Self::new(self.environment.clone()) + }; - self.initialize(round_height); - self.update_round_metrics(); + self.initialize(round_height); + self.update_round_metrics(); - Ok(Some(ResetCurrentRoundStorageAction { - remove_participants: Vec::new(), - })) - } else { - Ok(None) - } + Ok(ResetCurrentRoundStorageAction { + remove_participants: Vec::new(), + }) } /// @@ -2275,9 +2267,9 @@ impl CoordinatorState { } }; - match storage_action { - CeremonyStorageAction::ResetCurrentRound(_) => { - self.reset_round(time)?; + match &storage_action { + CeremonyStorageAction::ResetCurrentRound(_reset_action) => { + self.reset_current_round(time)?; } _ => {} } @@ -4431,7 +4423,7 @@ mod tests { let time = MockTimeSource::new(Utc::now()); - state.reset_round(&time).unwrap(); + let _action = state.reset_current_round(&time).unwrap(); // state.update_queue().unwrap(); From b13fd7a607376542c6f43bc9ee744a758cbbe1d0 Mon Sep 17 00:00:00 2001 From: Luke Frisken Date: Sun, 4 Jul 2021 16:50:26 +0400 Subject: [PATCH 11/18] documentation for reset_current_round --- phase1-coordinator/src/coordinator_state.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/phase1-coordinator/src/coordinator_state.rs b/phase1-coordinator/src/coordinator_state.rs index 35ebc5f7..909655fa 100644 --- a/phase1-coordinator/src/coordinator_state.rs +++ b/phase1-coordinator/src/coordinator_state.rs @@ -969,6 +969,9 @@ impl CoordinatorState { /// Reset the progress of the current round, back to how it was in /// its initialized state, however this does maintain the drop /// status of participants. + /// + /// Returns [CoordinatorError::RoundDoesNotExist] if + /// [CoordinatorState::current_round_height] is set to `None`. pub fn reset_current_round( &mut self, time: &dyn TimeSource, From 8919b54b2b33a3978ba736e89126fc55cd95acf7 Mon Sep 17 00:00:00 2001 From: Luke Frisken Date: Sun, 4 Jul 2021 17:46:18 +0400 Subject: [PATCH 12/18] work in progress round reset rollback --- phase1-coordinator/src/coordinator.rs | 38 +++-- phase1-coordinator/src/coordinator_state.rs | 170 ++++++++++++++------ phase1-coordinator/src/objects/round.rs | 4 +- 3 files changed, 148 insertions(+), 64 deletions(-) diff --git a/phase1-coordinator/src/coordinator.rs b/phase1-coordinator/src/coordinator.rs index 73bff127..f8e0ac11 100644 --- a/phase1-coordinator/src/coordinator.rs +++ b/phase1-coordinator/src/coordinator.rs @@ -5,7 +5,7 @@ use crate::{ authentication::Signature, commands::{Aggregation, Initialization}, - coordinator_state::{CoordinatorState, DropParticipant, ParticipantInfo, RoundMetrics}, + coordinator_state::{CeremonyStorageAction, CoordinatorState, DropParticipant, ParticipantInfo, RoundMetrics}, environment::{Deployment, Environment}, objects::{participant::*, task::TaskInitializationError, ContributionFileSignature, LockedLocators, Round, Task}, storage::{ContributionLocator, ContributionSignatureLocator, Locator, LocatorPath, Object, Storage, StorageLock}, @@ -2348,10 +2348,10 @@ impl Coordinator { }; match &drop_data.storage_action { - crate::coordinator_state::CeremonyStorageAction::ResetCurrentRound(_) => { - self.reset_round_storage(storage, vec![drop_data.participant.clone()])?; + CeremonyStorageAction::ResetCurrentRound(reset_action) => { + self.reset_round_storage(storage, &reset_action.remove_participants, reset_action.rollback)?; } - crate::coordinator_state::CeremonyStorageAction::ReplaceContributor(replace_action) => { + CeremonyStorageAction::ReplaceContributor(replace_action) => { warn!( "Replacing contributor {} with {}", replace_action.dropped_contributor, replace_action.replacement_contributor @@ -2393,7 +2393,7 @@ impl Coordinator { Object::RoundState(round), )?; } - crate::coordinator_state::CeremonyStorageAction::RemoveVerifier(remove_action) => { + CeremonyStorageAction::RemoveVerifier(remove_action) => { warn!("Removing verifier {}", remove_action.dropped_verifier); // Fetch the current round from storage. @@ -2486,18 +2486,26 @@ impl Coordinator { } /// Reset the current round in storage. + /// + /// + `remove_participants` is a list of participants that will + /// have their contributions removed from the round. + /// + If `rollback` is set to `true`, the current round will be + /// decremented by 1. If `rollback` is set to `true` and the + /// current round height is `0` then this will return an error + /// [CoordinatorError::RoundHeightIsZero] fn reset_round_storage( &self, storage: &mut StorageLock, - remove_participants: Vec, + remove_participants: &[Participant], + rollback: bool, ) -> Result<(), CoordinatorError> { // Fetch the current height of the ceremony. - let round_height = Self::load_current_round_height(storage)?; - let span = tracing::error_span!("reset_round", round = round_height); + let current_round_height = Self::load_current_round_height(storage)?; + let span = tracing::error_span!("reset_round", round = current_round_height); let _guard = span.enter(); - info!("Resetting round {}", round_height); + warn!("Resetting round {}", current_round_height); - let mut round = Self::load_round(storage, round_height)?; + let mut round = Self::load_round(storage, current_round_height)?; { // TODO: decide if we need a backup // if !storage.save_backup(backup_tag) { @@ -2528,7 +2536,15 @@ impl Coordinator { } } - info!("Finished resetting round {}", round_height); + if rollback { + if current_round_height == 0 { + return Err(CoordinatorError::RoundHeightIsZero); + } + + // TODO: perform the rollback + } + + warn!("Finished resetting round {}", current_round_height); Ok(()) } diff --git a/phase1-coordinator/src/coordinator_state.rs b/phase1-coordinator/src/coordinator_state.rs index 909655fa..1a9166f5 100644 --- a/phase1-coordinator/src/coordinator_state.rs +++ b/phase1-coordinator/src/coordinator_state.rs @@ -972,6 +972,9 @@ impl CoordinatorState { /// /// Returns [CoordinatorError::RoundDoesNotExist] if /// [CoordinatorState::current_round_height] is set to `None`. + /// + /// Returns [CoordinatorError::RoundHeightIsZero] if + /// [CoordinatorState::current_round_height] is set to `Some(0)`. pub fn reset_current_round( &mut self, time: &dyn TimeSource, @@ -979,67 +982,126 @@ impl CoordinatorState { let span = tracing::error_span!("reset_round", round = self.current_round_height.unwrap_or(0)); let _guard = span.enter(); - let round_height = self.current_round_height.ok_or(CoordinatorError::RoundDoesNotExist)?; - tracing::info!("Resetting round {}.", round_height); + let current_round_height = self.current_round_height.ok_or(CoordinatorError::RoundDoesNotExist)?; + if current_round_height == 0 { + return Err(CoordinatorError::RoundHeightIsZero); + } - // Fetch the number of chunks and bucket size. - let number_of_chunks = self.environment.number_of_chunks() as u64; + tracing::warn!("Resetting round {}.", current_round_height); - let prev_finished_contributors = self + let finished_contributors = self .finished_contributors - .get(&round_height) + .get(¤t_round_height) .cloned() .unwrap_or_else(|| HashMap::new()); - let number_of_contributors = self.current_contributors.len() + prev_finished_contributors.len(); - - let current_contributors = self - .current_contributors - .clone() - .into_iter() - .chain(prev_finished_contributors.into_iter()) - .enumerate() - .map(|(bucket_index, (participant, mut participant_info))| { - let bucket_id = bucket_index as u64; - let tasks = initialize_tasks(bucket_id, number_of_chunks, number_of_contributors as u64)?; - participant_info.restart_tasks(tasks, time)?; - Ok((participant, participant_info)) - }) - .collect::, CoordinatorError>>()?; - - let prev_finished_verifiers = self + let finished_verifiers = self .finished_verifiers - .get(&round_height) + .get(¤t_round_height) .cloned() .unwrap_or_else(|| HashMap::new()); - let current_verifiers = self - .current_verifiers - .clone() - .into_iter() - .chain(prev_finished_verifiers.into_iter()) - .map(|(participant, mut participant_info)| { - participant_info.restart_tasks(LinkedList::new(), time)?; - Ok((participant, participant_info)) + let number_of_contributors = self.current_contributors.len() + finished_contributors.len(); + let number_of_verifiers = self.current_verifiers.len() + finished_verifiers.len(); + let need_to_rollback = number_of_contributors == 0 || number_of_verifiers == 0; + + if need_to_rollback { + // Will roll back to the previous round and await new + // contributors/verifiers before starting the round again. + + if number_of_contributors == 0 { + tracing::warn!( + "No contributors remaining to reset and complete the current round. \ + Rolling back to wait and accept new participants." + ); + } + + if number_of_verifiers == 0 { + tracing::warn!( + "No verifiers remaining to reset and complete the current round. \ + Rolling back to wait and accept new participants." + ); + } + + let new_round_height = current_round_height - 1; + + let remove_participants: Vec = self + .current_contributors + .clone() + .into_iter() + .chain(finished_contributors.into_iter()) + .map(|(participant, _participant_info)| participant) + .collect(); + + *self = Self { + current_round_height: Some(new_round_height), + queue: self.queue.clone(), + banned: self.banned.clone(), + ..Self::new(self.environment.clone()) + }; + + self.initialize(new_round_height); + self.update_next_round_after(time); + + assert!(self.is_current_round_finished()); + assert!(self.is_current_round_aggregated()); + + // TODO there may be more things that we need to do here + // to get the coordinator into the waiting state. + + Ok(ResetCurrentRoundStorageAction { + remove_participants, + rollback: true, }) - .collect::, CoordinatorError>>()?; + } else { + // Will reset the round to run with the remaining participants. - *self = Self { - current_contributors, - current_verifiers, + // Fetch the number of chunks and bucket size. + let number_of_chunks = self.environment.number_of_chunks() as u64; - queue: self.queue.clone(), - banned: self.banned.clone(), - dropped: self.dropped.clone(), - ..Self::new(self.environment.clone()) - }; + let current_contributors = self + .current_contributors + .clone() + .into_iter() + .chain(finished_contributors.into_iter()) + .enumerate() + .map(|(bucket_index, (participant, mut participant_info))| { + let bucket_id = bucket_index as u64; + let tasks = initialize_tasks(bucket_id, number_of_chunks, number_of_contributors as u64)?; + participant_info.restart_tasks(tasks, time)?; + Ok((participant, participant_info)) + }) + .collect::, CoordinatorError>>()?; - self.initialize(round_height); - self.update_round_metrics(); + let current_verifiers = self + .current_verifiers + .clone() + .into_iter() + .chain(finished_verifiers.into_iter()) + .map(|(participant, mut participant_info)| { + participant_info.restart_tasks(LinkedList::new(), time)?; + Ok((participant, participant_info)) + }) + .collect::, CoordinatorError>>()?; - Ok(ResetCurrentRoundStorageAction { - remove_participants: Vec::new(), - }) + *self = Self { + current_contributors, + current_verifiers, + + queue: self.queue.clone(), + banned: self.banned.clone(), + dropped: self.dropped.clone(), + ..Self::new(self.environment.clone()) + }; + + self.initialize(current_round_height); + self.update_round_metrics(); + + Ok(ResetCurrentRoundStorageAction { + remove_participants: Vec::new(), + rollback: false, + }) + } } /// @@ -1961,7 +2023,7 @@ impl CoordinatorState { // Update the time to trigger the next round. if metrics.next_round_after.is_none() { - self.set_next_round_after(time); + self.update_next_round_after(time); } Ok(()) @@ -1969,7 +2031,7 @@ impl CoordinatorState { /// Set the `current_metrics` ([RoundMetrics]) `next_round_after` /// field to the appropriate value specified in the [Environment]. - fn set_next_round_after(&mut self, time: &dyn TimeSource) { + fn update_next_round_after(&mut self, time: &dyn TimeSource) { if let Some(metrics) = &mut self.current_metrics { metrics.next_round_after = Some(time.utc_now() + Duration::seconds(self.environment.queue_wait_time() as i64)); @@ -2200,6 +2262,7 @@ impl CoordinatorState { // There are no replacement contributors so the only option is to restart the round. CeremonyStorageAction::ResetCurrentRound(ResetCurrentRoundStorageAction { remove_participants: vec![participant.clone()], + rollback: true, }) } else { // TODO: handle the situation where all replacement contributors are currently engaged. @@ -2253,11 +2316,12 @@ impl CoordinatorState { // Restart the round because there are no verifiers // left for this round. - let reset_round = self.current_verifiers.is_empty(); + let reset_round = self.current_verifiers.is_empty() && self.finished_verifiers.is_empty(); if reset_round { CeremonyStorageAction::ResetCurrentRound(ResetCurrentRoundStorageAction { remove_participants: vec![participant.clone()], + rollback: false, }) } else { CeremonyStorageAction::RemoveVerifier(RemoveVerifierStorageAction { @@ -2272,7 +2336,9 @@ impl CoordinatorState { match &storage_action { CeremonyStorageAction::ResetCurrentRound(_reset_action) => { - self.reset_current_round(time)?; + // TODO: update the storage action if it now requires a rollback. + // Or think of some cleaner way to do this. + let _updated_action = self.reset_current_round(time)?; } _ => {} } @@ -3307,7 +3373,9 @@ impl CoordinatorState { pub struct ResetCurrentRoundStorageAction { /// The participants to be removed from the round during the /// reset. - remove_participants: Vec, + pub remove_participants: Vec, + /// Roll back to the previous round to await new participants. + pub rollback: bool, } /// Action to update the storage to reflect a verifier being diff --git a/phase1-coordinator/src/objects/round.rs b/phase1-coordinator/src/objects/round.rs index a81f75f6..be68543e 100644 --- a/phase1-coordinator/src/objects/round.rs +++ b/phase1-coordinator/src/objects/round.rs @@ -1070,7 +1070,7 @@ impl Round { /// the [crate::storage::Storage] to reflect the changes to the /// round state. `remove_participants` is a list of participants /// to remove from the round. - pub(crate) fn reset(&mut self, remove_participants: Vec) -> Vec { + pub(crate) fn reset(&mut self, remove_participants: &[Participant]) -> Vec { // TODO: in some cases it might be necessary to clear these, // such as when the round only consists of replacement // contributors. @@ -1197,7 +1197,7 @@ mod tests { let n_files = 2 * n_contributions + 2 * n_verifications; let n_actions = n_files + 1; // include action to update round - let actions = round_1.reset(vec![TEST_CONTRIBUTOR_ID_2.clone()]); + let actions = round_1.reset(&[TEST_CONTRIBUTOR_ID_2.clone()]); assert_eq!(n_actions, actions.len()); assert_eq!(64, round_1.chunks().len()); From acab4e7b435d64a5c67ff93cd297e521526c7120 Mon Sep 17 00:00:00 2001 From: Luke Frisken Date: Mon, 5 Jul 2021 12:57:51 +0400 Subject: [PATCH 13/18] work in progress implementation of round reset/rollback to wait for new participants --- phase1-coordinator/src/coordinator.rs | 20 ++- phase1-coordinator/src/coordinator_state.rs | 147 +++++++++++++------- 2 files changed, 112 insertions(+), 55 deletions(-) diff --git a/phase1-coordinator/src/coordinator.rs b/phase1-coordinator/src/coordinator.rs index f8e0ac11..57f25f06 100644 --- a/phase1-coordinator/src/coordinator.rs +++ b/phase1-coordinator/src/coordinator.rs @@ -1471,8 +1471,9 @@ impl Coordinator { /// /// Attempts to advance the ceremony to the next round. /// - #[inline] + #[tracing::instrument(skip(self, started_at))] pub fn try_advance(&self, started_at: DateTime) -> Result { + tracing::debug!("Trying to advance to the next round."); // Acquire the storage write lock. let mut storage = StorageLock::Write(self.storage.write().unwrap()); @@ -1483,16 +1484,23 @@ impl Coordinator { let current_round_height = { // Fetch the current round height from storage. let current_round_height_in_storage = Self::load_current_round_height(&storage)?; - trace!("Current round height in storage is {}", current_round_height_in_storage); + debug!("Current round height in storage is {}", current_round_height_in_storage); // Fetch the current round height from coordinator state. let current_round_height = state.current_round_height(); - trace!("Current round height in coordinator state is {}", current_round_height); + debug!("Current round height in coordinator state is {}", current_round_height); // Check that the current round height matches in storage and state. - match current_round_height_in_storage == current_round_height { - true => current_round_height, - false => return Err(CoordinatorError::RoundHeightMismatch), + if current_round_height_in_storage == current_round_height { + current_round_height + } else { + tracing::error!( + "Round height in storage ({}) does not match the \ + round height in coordinator state ({})", + current_round_height_in_storage, + current_round_height, + ); + return Err(CoordinatorError::RoundHeightMismatch); } }; diff --git a/phase1-coordinator/src/coordinator_state.rs b/phase1-coordinator/src/coordinator_state.rs index 1a9166f5..abbf4118 100644 --- a/phase1-coordinator/src/coordinator_state.rs +++ b/phase1-coordinator/src/coordinator_state.rs @@ -1003,28 +1003,56 @@ impl CoordinatorState { let number_of_contributors = self.current_contributors.len() + finished_contributors.len(); let number_of_verifiers = self.current_verifiers.len() + finished_verifiers.len(); + let number_of_chunks = self.environment.number_of_chunks() as u64; + + let current_contributors = self + .current_contributors + .clone() + .into_iter() + .chain(finished_contributors.clone().into_iter()) + .enumerate() + .map(|(bucket_index, (participant, mut participant_info))| { + let bucket_id = bucket_index as u64; + let tasks = initialize_tasks(bucket_id, number_of_chunks, number_of_contributors as u64)?; + participant_info.restart_tasks(tasks, time)?; + Ok((participant, participant_info)) + }) + .collect::, CoordinatorError>>()?; + + let current_verifiers = self + .current_verifiers + .clone() + .into_iter() + .chain(finished_verifiers.clone().into_iter()) + .map(|(participant, mut participant_info)| { + participant_info.restart_tasks(LinkedList::new(), time)?; + Ok((participant, participant_info)) + }) + .collect::, CoordinatorError>>()?; + let need_to_rollback = number_of_contributors == 0 || number_of_verifiers == 0; if need_to_rollback { // Will roll back to the previous round and await new // contributors/verifiers before starting the round again. + let new_round_height = current_round_height - 1; if number_of_contributors == 0 { tracing::warn!( "No contributors remaining to reset and complete the current round. \ - Rolling back to wait and accept new participants." + Rolling back to round {} to wait and accept new participants.", + new_round_height ); } if number_of_verifiers == 0 { tracing::warn!( "No verifiers remaining to reset and complete the current round. \ - Rolling back to wait and accept new participants." + Rolling back to round {} to wait and accept new participants.", + new_round_height ); } - let new_round_height = current_round_height - 1; - let remove_participants: Vec = self .current_contributors .clone() @@ -1033,9 +1061,31 @@ impl CoordinatorState { .map(|(participant, _participant_info)| participant) .collect(); + let current_metrics = Some(RoundMetrics { + is_round_aggregated: true, + started_aggregation_at: Some(time.utc_now()), + finished_aggregation_at: Some(time.utc_now()), + ..Default::default() + }); + + let mut queue = self.queue.clone(); + + // Add each participant back into the queue. + for (participant, participant_info) in current_contributors + .iter() + .chain(self.current_verifiers.iter()) + .chain(self.next.iter()) + { + queue.insert( + participant.clone(), + (participant_info.reliability, Some(participant_info.round_height)), + ); + } + *self = Self { + current_metrics, current_round_height: Some(new_round_height), - queue: self.queue.clone(), + queue, banned: self.banned.clone(), ..Self::new(self.environment.clone()) }; @@ -1043,8 +1093,24 @@ impl CoordinatorState { self.initialize(new_round_height); self.update_next_round_after(time); - assert!(self.is_current_round_finished()); - assert!(self.is_current_round_aggregated()); + if !self.is_current_round_finished() { + tracing::error!( + "Round rollback was not properly completed, \ + the current round is not finished." + ); + } + + if !self.is_current_round_aggregated() { + tracing::error!( + "Round rollback was not properly completed, \ + the current round is not aggregated." + ); + } + + tracing::info!( + "Completed rollback to round {}, now awaiting new participants.", + new_round_height + ); // TODO there may be more things that we need to do here // to get the coordinator into the waiting state. @@ -1056,34 +1122,6 @@ impl CoordinatorState { } else { // Will reset the round to run with the remaining participants. - // Fetch the number of chunks and bucket size. - let number_of_chunks = self.environment.number_of_chunks() as u64; - - let current_contributors = self - .current_contributors - .clone() - .into_iter() - .chain(finished_contributors.into_iter()) - .enumerate() - .map(|(bucket_index, (participant, mut participant_info))| { - let bucket_id = bucket_index as u64; - let tasks = initialize_tasks(bucket_id, number_of_chunks, number_of_contributors as u64)?; - participant_info.restart_tasks(tasks, time)?; - Ok((participant, participant_info)) - }) - .collect::, CoordinatorError>>()?; - - let current_verifiers = self - .current_verifiers - .clone() - .into_iter() - .chain(finished_verifiers.into_iter()) - .map(|(participant, mut participant_info)| { - participant_info.restart_tasks(LinkedList::new(), time)?; - Ok((participant, participant_info)) - }) - .collect::, CoordinatorError>>()?; - *self = Self { current_contributors, current_verifiers, @@ -2334,18 +2372,26 @@ impl CoordinatorState { } }; - match &storage_action { - CeremonyStorageAction::ResetCurrentRound(_reset_action) => { - // TODO: update the storage action if it now requires a rollback. - // Or think of some cleaner way to do this. - let _updated_action = self.reset_current_round(time)?; + // Perform the round reset if we need to. + let final_storage_action = match storage_action { + CeremonyStorageAction::ResetCurrentRound(mut reset_action) => { + let extra_reset_action = self.reset_current_round(time)?; + // Extend the reset action with any requirements from reset_current_round + for participant in extra_reset_action.remove_participants { + if !reset_action.remove_participants.contains(&participant) { + reset_action.remove_participants.push(participant) + } + } + // Rollback takes priority. + reset_action.rollback |= extra_reset_action.rollback; + CeremonyStorageAction::ResetCurrentRound(reset_action) } - _ => {} - } + action => action, + }; let drop_data = DropCurrentParticpantData { participant: participant.clone(), - storage_action, + storage_action: final_storage_action, }; Ok(DropParticipant::DropCurrent(drop_data)) @@ -2881,13 +2927,13 @@ impl CoordinatorState { /// Prepares transition of the coordinator state from the current round to the next round. /// On precommit success, returns the list of contributors and verifiers for the next round. /// - #[inline] + #[tracing::instrument(skip(self, time))] pub(super) fn precommit_next_round( &mut self, next_round_height: u64, time: &dyn TimeSource, ) -> Result<(Vec, Vec), CoordinatorError> { - trace!("Attempting to run precommit for round {}", next_round_height); + tracing::debug!("Attempting to run precommit for round {}", next_round_height); // Check that the coordinator state is initialized. if self.status == CoordinatorStatus::Initializing { @@ -4641,9 +4687,12 @@ mod tests { DropParticipant::DropQueue(_) => panic!("Unexpected drop type: {:?}", drop), }; - assert!(matches!( - drop_data.storage_action, - CeremonyStorageAction::ResetCurrentRound(_) - )); + match drop_data.storage_action { + CeremonyStorageAction::ResetCurrentRound(reset_action) => { + dbg!(reset_action.remove_participants); + assert!(reset_action.rollback) + } + unexpected => panic!("unexpected storage action: {:?}", unexpected), + } } } From debab47bcc9a73441e02ab2e663d63af205c6eca Mon Sep 17 00:00:00 2001 From: Luke Frisken Date: Thu, 8 Jul 2021 14:07:22 +0400 Subject: [PATCH 14/18] fix rolling back to previous round during round reset --- phase1-coordinator/src/coordinator.rs | 45 ++++++++++----------------- 1 file changed, 16 insertions(+), 29 deletions(-) diff --git a/phase1-coordinator/src/coordinator.rs b/phase1-coordinator/src/coordinator.rs index 57f25f06..f82c7387 100644 --- a/phase1-coordinator/src/coordinator.rs +++ b/phase1-coordinator/src/coordinator.rs @@ -2514,34 +2514,15 @@ impl Coordinator { warn!("Resetting round {}", current_round_height); let mut round = Self::load_round(storage, current_round_height)?; + + tracing::debug!("Resetting round and applying storage changes"); + if let Some(error) = round + .reset(remove_participants) + .into_iter() + .map(|action| storage.process(action)) + .find_map(Result::err) { - // TODO: decide if we need a backup - // if !storage.save_backup(backup_tag) { - // error!( - // "Could not save storage backup for round {} with tag {}", - // round_height, backup_tag - // ); - // return Err(CoordinatorError::StorageUpdateFailed); - // } - - // if let Err(error) = storage.remove(&Locator::RoundHeight) { - // error!("Could not remove round height from storage because: {}", error); - // return Err(CoordinatorError::StorageUpdateFailed); - // } - // if let Err(error) = storage.insert(Locator::RoundHeight, Object::RoundHeight(round_height - 1)) { - // error!("Could not insert round height to storage because: {}", error); - // return Err(CoordinatorError::StorageUpdateFailed); - // } - - tracing::debug!("Resetting round and applying storage changes"); - if let Some(error) = round - .reset(remove_participants) - .into_iter() - .map(|action| storage.process(action)) - .find_map(Result::err) - { - return Err(error); - } + return Err(error); } if rollback { @@ -2549,10 +2530,16 @@ impl Coordinator { return Err(CoordinatorError::RoundHeightIsZero); } - // TODO: perform the rollback + let new_round_height = current_round_height - 1; + tracing::debug!("Rolling back to round {} in storage.", new_round_height); + + storage.remove(&Locator::RoundState { + round_height: current_round_height, + })?; + storage.update(&Locator::RoundHeight, Object::RoundHeight(new_round_height))?; } - warn!("Finished resetting round {}", current_round_height); + warn!("Finished resetting round {} storage", current_round_height); Ok(()) } From 8459ef7e06e75366c8d0bd56278149cc15abe411 Mon Sep 17 00:00:00 2001 From: Luke Frisken Date: Thu, 8 Jul 2021 21:55:42 +0400 Subject: [PATCH 15/18] fix small bug with going into rollback when it shouldn't, extra unit test, more docs --- phase1-coordinator/src/coordinator_state.rs | 216 ++++++++++++++++++-- 1 file changed, 195 insertions(+), 21 deletions(-) diff --git a/phase1-coordinator/src/coordinator_state.rs b/phase1-coordinator/src/coordinator_state.rs index abbf4118..765ac166 100644 --- a/phase1-coordinator/src/coordinator_state.rs +++ b/phase1-coordinator/src/coordinator_state.rs @@ -2101,13 +2101,23 @@ impl CoordinatorState { } /// - /// Drops the given participant from the queue, precommit, and current round. + /// Drops the given participant from the queue, precommit, and + /// current round. /// - /// The dropped participant information preserves the state of locked chunks, - /// pending tasks, and completed tasks, as reference in case this state is - /// necessary in the future. + /// Returns information/actions for the coordinator to perform in + /// response to the participant being dropped in the form of + /// [DropParticipant]. /// - /// On success, this function returns a `Justification` for the coordinator to use. + /// If the participant is a [Participant::Contributor], this will + /// attempt to replace the contributor with an available + /// replacement contributor. If there are no replacement + /// contributors available the round will be reset via + /// [CoordinatorState::reset_current_round]. + /// + /// If the participant being dropped is the only remaining regular + /// (non-replacement) contributor or the only remaining verifier, + /// the reset will include a rollback to wait for new participants + /// before restarting the round again. /// #[tracing::instrument( skip(self, participant, time), @@ -2300,7 +2310,7 @@ impl CoordinatorState { // There are no replacement contributors so the only option is to restart the round. CeremonyStorageAction::ResetCurrentRound(ResetCurrentRoundStorageAction { remove_participants: vec![participant.clone()], - rollback: true, + rollback: false, }) } else { // TODO: handle the situation where all replacement contributors are currently engaged. @@ -3489,8 +3499,8 @@ pub(crate) struct DropQueueParticipantData { pub participant: Participant, } -/// The reason for dropping a participant and the and data needed to -/// perform the drop. +/// Returns information/actions for the coordinator to perform in +/// response to the participant being dropped. #[derive(Debug)] pub(crate) enum DropParticipant { /// Coordinator has decided that a participant needs to be dropped @@ -4442,12 +4452,16 @@ mod tests { assert_eq!(0, state.banned.len()); } + /// Test a manually triggered round reset during a round with two + /// contributors and two verifiers. #[test] fn test_round_reset() { test_logger(); let time = SystemTimeSource::new(); - let environment = TEST_ENVIRONMENT.clone(); + let environment: Environment = Testing::from(Parameters::Test8Chunks) + .coordinator_contributors(&[]) + .into(); // Fetch two contributors and two verifiers. let contributor_1 = TEST_CONTRIBUTOR_ID.clone(); @@ -4540,9 +4554,8 @@ mod tests { let time = MockTimeSource::new(Utc::now()); - let _action = state.reset_current_round(&time).unwrap(); - - // state.update_queue().unwrap(); + let action = state.reset_current_round(&time).unwrap(); + assert!(!action.rollback); for _ in 0..number_of_chunks { // Contributor 1 @@ -4608,9 +4621,171 @@ mod tests { assert!(state.is_current_round_finished()); } - /// Test round reset when all contributors have been dropped. + /// Test a round reset triggered by a drop of one contributor + /// during a round with two contributors and two verifiers. The + /// reset is triggered because there are no replacement + /// contributors. #[test] - fn test_round_reset_drop_all_contributors() { + fn test_round_reset_drop_one_contributor() { + test_logger(); + + let time = SystemTimeSource::new(); + let environment: Environment = Testing::from(Parameters::Test8Chunks) + .coordinator_contributors(&[]) + .into(); + + // Fetch two contributors and two verifiers. + let contributor_1 = TEST_CONTRIBUTOR_ID.clone(); + let contributor_2 = TEST_CONTRIBUTOR_ID_2.clone(); + let verifier_1 = TEST_VERIFIER_ID.clone(); + let verifier_2 = TEST_VERIFIER_ID_2.clone(); + + // Initialize a new coordinator state. + let current_round_height = 5; + let mut state = CoordinatorState::new(environment.clone()); + state.initialize(current_round_height); + state.add_to_queue(contributor_1.clone(), 10).unwrap(); + state.add_to_queue(contributor_2.clone(), 9).unwrap(); + state.add_to_queue(verifier_1.clone(), 10).unwrap(); + state.add_to_queue(verifier_2.clone(), 9).unwrap(); + state.update_queue().unwrap(); + state.aggregating_current_round(&time).unwrap(); + state.aggregated_current_round(&time).unwrap(); + + // Advance the coordinator to the next round. + let next_round_height = current_round_height + 1; + state.precommit_next_round(next_round_height, &time).unwrap(); + state.commit_next_round(); + + let number_of_chunks = environment.number_of_chunks(); + let chunks_3_4: u64 = (number_of_chunks * 3) / 4; + + for _ in 0..chunks_3_4 { + // Contributor 1 + { + // Fetch a pending task for the contributor. + let task = state.fetch_task(&contributor_1, &time).unwrap(); + state.acquired_lock(&contributor_1, task.chunk_id(), &time).unwrap(); + let assigned_verifier = state.completed_task(&contributor_1, task, &time).unwrap(); + // Fetch a pending task for the verifier. + let task = state.fetch_task(&assigned_verifier, &time).unwrap(); + + state.acquired_lock(&assigned_verifier, task.chunk_id(), &time).unwrap(); + state.completed_task(&assigned_verifier, task, &time).unwrap(); + + { + // Update the current round metrics. + state.update_round_metrics(); + + // Update the state of current round contributors. + state.update_current_contributors(&time).unwrap(); + + // Update the state of current round verifiers. + state.update_current_verifiers(&time).unwrap(); + + // Drop disconnected participants from the current round. + state.update_dropped_participants(&time).unwrap(); + + // Ban any participants who meet the coordinator criteria. + state.update_banned_participants().unwrap(); + } + } + + // Contributor 2 + { + // Fetch a pending task for the contributor. + let task = state.fetch_task(&contributor_2, &time).unwrap(); + state.acquired_lock(&contributor_2, task.chunk_id(), &time).unwrap(); + let assigned_verifier = state.completed_task(&contributor_2, task, &time).unwrap(); + // Fetch a pending task for the verifier. + let task = state.fetch_task(&assigned_verifier, &time).unwrap(); + state.acquired_lock(&assigned_verifier, task.chunk_id(), &time).unwrap(); + state.completed_task(&assigned_verifier, task, &time).unwrap(); + + { + // Update the current round metrics. + state.update_round_metrics(); + + // Update the state of current round contributors. + state.update_current_contributors(&time).unwrap(); + + // Update the state of current round verifiers. + state.update_current_verifiers(&time).unwrap(); + + // Drop disconnected participants from the current round. + state.update_dropped_participants(&time).unwrap(); + + // Ban any participants who meet the coordinator criteria. + state.update_banned_participants().unwrap(); + } + } + } + + assert!(!state.is_current_round_finished()); + + let time = MockTimeSource::new(Utc::now()); + + dbg!(&state.current_contributors()); + + let drop = state.drop_participant(&contributor_1, &time).unwrap(); + + dbg!(&state.current_contributors()); + + let drop_data = match drop { + DropParticipant::DropCurrent(drop_data) => drop_data, + DropParticipant::DropQueue(_) => panic!("Unexpected drop type: {:?}", drop), + }; + + let reset_action = match drop_data.storage_action { + CeremonyStorageAction::ResetCurrentRound(reset_action) => reset_action, + unexpected => panic!("unexpected storage action: {:?}", unexpected), + }; + + assert_eq!(1, reset_action.remove_participants.len()); + assert!(!reset_action.rollback); + + for _ in 0..number_of_chunks { + // Contributor 2 + { + // Fetch a pending task for the contributor. + let task = state.fetch_task(&contributor_2, &time).unwrap(); + state.acquired_lock(&contributor_2, task.chunk_id(), &time).unwrap(); + let assigned_verifier = state.completed_task(&contributor_2, task, &time).unwrap(); + // Fetch a pending task for the verifier. + let task = state.fetch_task(&assigned_verifier, &time).unwrap(); + state.acquired_lock(&assigned_verifier, task.chunk_id(), &time).unwrap(); + state.completed_task(&assigned_verifier, task, &time).unwrap(); + + { + // Update the current round metrics. + state.update_round_metrics(); + + // Update the state of current round contributors. + state.update_current_contributors(&time).unwrap(); + + // Update the state of current round verifiers. + state.update_current_verifiers(&time).unwrap(); + + // Drop disconnected participants from the current round. + state.update_dropped_participants(&time).unwrap(); + + // Ban any participants who meet the coordinator criteria. + state.update_banned_participants().unwrap(); + } + } + } + + assert!(state.is_current_round_finished()); + } + + /// Test round reset when all contributors have been dropped + /// during a round that has two contributors and two verifiers. + /// The reset is triggered because there are no replacement + /// contributors, and the reset includes a rollback to invite new + /// contributors because there are no contributors remaining in + /// the round. + #[test] + fn test_round_reset_rollback_drop_all_contributors() { test_logger(); let time = SystemTimeSource::new(); @@ -4619,7 +4794,6 @@ mod tests { let environment: Environment = Testing::from(Parameters::Test8Chunks) .coordinator_contributors(&[]) .into(); - dbg!(environment.coordinator_contributors()); // Fetch two contributors and two verifiers. let contributor_1 = TEST_CONTRIBUTOR_ID.clone(); @@ -4687,12 +4861,12 @@ mod tests { DropParticipant::DropQueue(_) => panic!("Unexpected drop type: {:?}", drop), }; - match drop_data.storage_action { - CeremonyStorageAction::ResetCurrentRound(reset_action) => { - dbg!(reset_action.remove_participants); - assert!(reset_action.rollback) - } + let reset_action = match drop_data.storage_action { + CeremonyStorageAction::ResetCurrentRound(reset_action) => reset_action, unexpected => panic!("unexpected storage action: {:?}", unexpected), - } + }; + + assert_eq!(1, reset_action.remove_participants.len()); + assert!(reset_action.rollback) } } From 2cafd2f503e34c704f5747166524008babfa7b0c Mon Sep 17 00:00:00 2001 From: Luke Frisken Date: Mon, 12 Jul 2021 16:39:03 +0400 Subject: [PATCH 16/18] changes for admin/manual round restart --- phase1-coordinator/src/coordinator.rs | 35 +++++++++++++++++---- phase1-coordinator/src/coordinator_state.rs | 14 +++++++-- phase1-coordinator/src/objects/round.rs | 6 ---- phase1-coordinator/src/storage/storage.rs | 23 ++++++++++---- 4 files changed, 57 insertions(+), 21 deletions(-) diff --git a/phase1-coordinator/src/coordinator.rs b/phase1-coordinator/src/coordinator.rs index f82c7387..e68f3c72 100644 --- a/phase1-coordinator/src/coordinator.rs +++ b/phase1-coordinator/src/coordinator.rs @@ -5,7 +5,14 @@ use crate::{ authentication::Signature, commands::{Aggregation, Initialization}, - coordinator_state::{CeremonyStorageAction, CoordinatorState, DropParticipant, ParticipantInfo, RoundMetrics}, + coordinator_state::{ + CeremonyStorageAction, + CoordinatorState, + DropParticipant, + ParticipantInfo, + ResetCurrentRoundStorageAction, + RoundMetrics, + }, environment::{Deployment, Environment}, objects::{participant::*, task::TaskInitializationError, ContributionFileSignature, LockedLocators, Round, Task}, storage::{ContributionLocator, ContributionSignatureLocator, Locator, LocatorPath, Object, Storage, StorageLock}, @@ -2357,7 +2364,7 @@ impl Coordinator { match &drop_data.storage_action { CeremonyStorageAction::ResetCurrentRound(reset_action) => { - self.reset_round_storage(storage, &reset_action.remove_participants, reset_action.rollback)?; + self.reset_round_storage(storage, reset_action)?; } CeremonyStorageAction::ReplaceContributor(replace_action) => { warn!( @@ -2493,6 +2500,23 @@ impl Coordinator { self.signature.clone() } + /// + /// Resets the current round, with a rollback to the end of the + /// previous round to invite new participants into the round. + /// + pub fn reset_round(&self) -> Result<(), CoordinatorError> { + let mut state = self.state.write().map_err(|_| CoordinatorError::StateLockFailed)?; + let reset_action = state.reset_current_round(true, &*self.time)?; + + // Acquire the storage write lock. + let mut storage = StorageLock::Write(self.storage.write().map_err(|_| CoordinatorError::StorageLockFailed)?); + + storage.update(&Locator::CoordinatorState, Object::CoordinatorState(state.clone()))?; + self.reset_round_storage(&mut storage, &reset_action)?; + + Ok(()) + } + /// Reset the current round in storage. /// /// + `remove_participants` is a list of participants that will @@ -2504,8 +2528,7 @@ impl Coordinator { fn reset_round_storage( &self, storage: &mut StorageLock, - remove_participants: &[Participant], - rollback: bool, + reset_action: &ResetCurrentRoundStorageAction, ) -> Result<(), CoordinatorError> { // Fetch the current height of the ceremony. let current_round_height = Self::load_current_round_height(storage)?; @@ -2517,7 +2540,7 @@ impl Coordinator { tracing::debug!("Resetting round and applying storage changes"); if let Some(error) = round - .reset(remove_participants) + .reset(&reset_action.remove_participants) .into_iter() .map(|action| storage.process(action)) .find_map(Result::err) @@ -2525,7 +2548,7 @@ impl Coordinator { return Err(error); } - if rollback { + if reset_action.rollback { if current_round_height == 0 { return Err(CoordinatorError::RoundHeightIsZero); } diff --git a/phase1-coordinator/src/coordinator_state.rs b/phase1-coordinator/src/coordinator_state.rs index 765ac166..b71f8d8a 100644 --- a/phase1-coordinator/src/coordinator_state.rs +++ b/phase1-coordinator/src/coordinator_state.rs @@ -970,6 +970,13 @@ impl CoordinatorState { /// its initialized state, however this does maintain the drop /// status of participants. /// + /// If `force_rollback` is set to `true` the coordinator will be + /// forced to reset to the end of the previous round and begin + /// waiting for new participants again before restarting the + /// current round. If set to `false` the rollback will only occur + /// if there are no available contributors or no available + /// verifiers left in the round. + /// /// Returns [CoordinatorError::RoundDoesNotExist] if /// [CoordinatorState::current_round_height] is set to `None`. /// @@ -977,6 +984,7 @@ impl CoordinatorState { /// [CoordinatorState::current_round_height] is set to `Some(0)`. pub fn reset_current_round( &mut self, + force_rollback: bool, time: &dyn TimeSource, ) -> Result { let span = tracing::error_span!("reset_round", round = self.current_round_height.unwrap_or(0)); @@ -1030,7 +1038,7 @@ impl CoordinatorState { }) .collect::, CoordinatorError>>()?; - let need_to_rollback = number_of_contributors == 0 || number_of_verifiers == 0; + let need_to_rollback = force_rollback || number_of_contributors == 0 || number_of_verifiers == 0; if need_to_rollback { // Will roll back to the previous round and await new @@ -2385,7 +2393,7 @@ impl CoordinatorState { // Perform the round reset if we need to. let final_storage_action = match storage_action { CeremonyStorageAction::ResetCurrentRound(mut reset_action) => { - let extra_reset_action = self.reset_current_round(time)?; + let extra_reset_action = self.reset_current_round(false, time)?; // Extend the reset action with any requirements from reset_current_round for participant in extra_reset_action.remove_participants { if !reset_action.remove_participants.contains(&participant) { @@ -4554,7 +4562,7 @@ mod tests { let time = MockTimeSource::new(Utc::now()); - let action = state.reset_current_round(&time).unwrap(); + let action = state.reset_current_round(false, &time).unwrap(); assert!(!action.rollback); for _ in 0..number_of_chunks { diff --git a/phase1-coordinator/src/objects/round.rs b/phase1-coordinator/src/objects/round.rs index be68543e..136057fb 100644 --- a/phase1-coordinator/src/objects/round.rs +++ b/phase1-coordinator/src/objects/round.rs @@ -1071,12 +1071,6 @@ impl Round { /// round state. `remove_participants` is a list of participants /// to remove from the round. pub(crate) fn reset(&mut self, remove_participants: &[Participant]) -> Vec { - // TODO: in some cases it might be necessary to clear these, - // such as when the round only consists of replacement - // contributors. - // self.contributor_ids.clear(); - // self.verifier_ids.clear(); - let mut actions: Vec = self .chunks .iter_mut() diff --git a/phase1-coordinator/src/storage/storage.rs b/phase1-coordinator/src/storage/storage.rs index 0c2c0d25..43b590ce 100644 --- a/phase1-coordinator/src/storage/storage.rs +++ b/phase1-coordinator/src/storage/storage.rs @@ -319,7 +319,7 @@ impl TryFrom<&Path> for LocatorPath { /// /// **Note:** This can probably be refactored out in the future so /// that we only use [Locator]. -#[derive(Clone, PartialEq)] +#[derive(Clone, PartialEq, Debug)] pub enum LocatorOrPath { Path(LocatorPath), Locator(Locator), @@ -332,6 +332,13 @@ impl LocatorOrPath { LocatorOrPath::Locator(locator) => Ok(locator), } } + + pub fn try_into_path(self, storage: &impl Storage) -> Result { + match self { + LocatorOrPath::Path(path) => Ok(path), + LocatorOrPath::Locator(locator) => storage.to_path(&locator), + } + } } impl From for LocatorOrPath { @@ -347,29 +354,33 @@ impl From for LocatorOrPath { } /// An action to remove an item from [Storage]. -#[derive(Clone, PartialEq)] +#[derive(Clone, PartialEq, Debug)] pub struct RemoveAction { - locator: LocatorOrPath, + locator_or_path: LocatorOrPath, } impl RemoveAction { /// Create a new [RemoveAction] pub fn new(locator: impl Into) -> Self { Self { - locator: locator.into(), + locator_or_path: locator.into(), } } /// Obtain the location of the item to be removed from [Storage] /// as a [LocatorOrPath]. pub fn locator_or_path(&self) -> &LocatorOrPath { - &self.locator + &self.locator_or_path } /// Obtain the location of the item to be removed from [Storage] /// as a [Locator]. pub fn try_into_locator(self, storage: &impl Storage) -> Result { - self.locator.try_into_locator(storage) + self.locator_or_path.try_into_locator(storage) + } + + pub fn try_into_path(self, storage: &impl Storage) -> Result { + self.locator_or_path.try_into_path(storage) } } From 5338f49c6838394469ac657a061657bdddcc0d61 Mon Sep 17 00:00:00 2001 From: Luke Frisken Date: Tue, 13 Jul 2021 22:10:05 +0400 Subject: [PATCH 17/18] fix failing unit tests, contributor id wasn't being removed from the round during drop --- phase1-coordinator/src/coordinator.rs | 14 ++----------- phase1-coordinator/src/objects/round.rs | 28 +++++++++++++++++++++++++ 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/phase1-coordinator/src/coordinator.rs b/phase1-coordinator/src/coordinator.rs index e68f3c72..5cce28da 100644 --- a/phase1-coordinator/src/coordinator.rs +++ b/phase1-coordinator/src/coordinator.rs @@ -2375,23 +2375,13 @@ impl Coordinator { // Fetch the current round from storage. let mut round = Self::load_current_round(&storage)?; - warn!("Removing locked chunks and all impacted contributions"); - - // Remove the lock from the specified chunks. - round.remove_locks_unsafe( + // Remove the contributor from the round state. + round.remove_contributor_unsafe( storage, &replace_action.dropped_contributor, &replace_action.locked_chunks, - )?; - warn!("Removed locked chunks"); - - // Remove the contributions from the specified chunks. - round.remove_chunk_contributions_unsafe( - storage, - &replace_action.dropped_contributor, &replace_action.tasks, )?; - warn!("Removed impacted contributions"); // Assign a replacement contributor to the dropped tasks for the current round. round.add_replacement_contributor_unsafe(replace_action.replacement_contributor.clone())?; diff --git a/phase1-coordinator/src/objects/round.rs b/phase1-coordinator/src/objects/round.rs index 136057fb..7a392e42 100644 --- a/phase1-coordinator/src/objects/round.rs +++ b/phase1-coordinator/src/objects/round.rs @@ -794,6 +794,34 @@ impl Round { } } + /// Remove a contributor from the round. + pub(crate) fn remove_contributor_unsafe( + &mut self, + storage: &mut StorageLock, + contributor: &Participant, + locked_chunks: &[u64], + tasks: &[Task], + ) -> Result<(), CoordinatorError> { + warn!("Removing locked chunks and all impacted contributions"); + + // Remove the lock from the specified chunks. + self.remove_locks_unsafe(storage, contributor, locked_chunks)?; + warn!("Removed locked chunks"); + + // Remove the contributions from the specified chunks. + self.remove_chunk_contributions_unsafe(storage, contributor, tasks)?; + warn!("Removed impacted contributions"); + + self.contributor_ids = self + .contributor_ids + .clone() + .into_iter() + .filter(|participant| participant != contributor) + .collect(); + + Ok(()) + } + /// /// Removes the locks for the current round from the given chunk IDs. /// From 8e625e89537b9ba023c7f1082db514e324c33c3e Mon Sep 17 00:00:00 2001 From: Luke Frisken Date: Sat, 17 Jul 2021 11:08:15 +0400 Subject: [PATCH 18/18] update tokio to fix audit warning for https://rustsec.org/advisories/RUSTSEC-2021-0072 --- Cargo.lock | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 75ee3a00..26075e10 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1494,7 +1494,7 @@ dependencies = [ "http", "indexmap", "slab", - "tokio 1.7.0", + "tokio 1.8.1", "tokio-util 0.6.7", "tracing", ] @@ -1662,7 +1662,7 @@ dependencies = [ "itoa", "pin-project 1.0.2", "socket2 0.4.0", - "tokio 1.7.0", + "tokio 1.8.1", "tower-service", "tracing", "want", @@ -1695,7 +1695,7 @@ dependencies = [ "bytes 1.0.1", "hyper 0.14.7", "native-tls", - "tokio 1.7.0", + "tokio 1.8.1", "tokio-native-tls", ] @@ -2519,7 +2519,7 @@ dependencies = [ "serial_test", "setup-utils", "thiserror", - "tokio 1.7.0", + "tokio 1.8.1", "tracing", "tracing-subscriber", ] @@ -3008,7 +3008,7 @@ dependencies = [ "pin-project-lite 0.2.6", "serde", "serde_urlencoded", - "tokio 1.7.0", + "tokio 1.8.1", "tokio-native-tls", "url", "wasm-bindgen", @@ -3554,7 +3554,7 @@ dependencies = [ "snarkos-toolkit", "structopt", "thiserror", - "tokio 1.7.0", + "tokio 1.8.1", "tokio-tungstenite", "tracing", "tracing-appender", @@ -3568,7 +3568,7 @@ version = "0.1.0" dependencies = [ "serde", "serde_json", - "tokio 1.7.0", + "tokio 1.8.1", ] [[package]] @@ -3598,7 +3598,7 @@ dependencies = [ "snarkos-toolkit", "structopt", "thiserror", - "tokio 1.7.0", + "tokio 1.8.1", "tracing", "tracing-subscriber", "url", @@ -4447,9 +4447,9 @@ dependencies = [ [[package]] name = "tokio" -version = "1.7.0" +version = "1.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c79ba603c337335df6ba6dd6afc38c38a7d5e1b0c871678439ea973cd62a118e" +checksum = "98c8b05dc14c75ea83d63dd391100353789f5f24b8b3866542a5e85c8be8e985" dependencies = [ "autocfg", "bytes 1.0.1", @@ -4482,7 +4482,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f7d995660bd2b7f8c1568414c1126076c13fbb725c40112dc0120b78eb9b717b" dependencies = [ "native-tls", - "tokio 1.7.0", + "tokio 1.8.1", ] [[package]] @@ -4506,7 +4506,7 @@ dependencies = [ "futures-util", "log", "pin-project 1.0.2", - "tokio 1.7.0", + "tokio 1.8.1", "tungstenite", ] @@ -4535,7 +4535,7 @@ dependencies = [ "futures-sink", "log", "pin-project-lite 0.2.6", - "tokio 1.7.0", + "tokio 1.8.1", ] [[package]]