From c34667a39d240ac3febd75a41e17a5e2dcc37ab6 Mon Sep 17 00:00:00 2001 From: Damien LACHAUME / PALO-IT Date: Tue, 11 Jul 2023 19:40:35 +0200 Subject: [PATCH] Move end-to-end assertions to a module & add run-only option to e2e --- .../mithril-end-to-end/src/assertions/mod.rs | 376 +++++++++++++++ .../mithril-end-to-end/src/end_to_end_spec.rs | 440 ++---------------- .../mithril-end-to-end/src/lib.rs | 3 + .../mithril-end-to-end/src/main.rs | 53 ++- .../src/mithril/infrastructure.rs | 7 + .../mithril-end-to-end/src/run_only.rs | 87 ++++ 6 files changed, 549 insertions(+), 417 deletions(-) create mode 100644 mithril-test-lab/mithril-end-to-end/src/assertions/mod.rs create mode 100644 mithril-test-lab/mithril-end-to-end/src/run_only.rs diff --git a/mithril-test-lab/mithril-end-to-end/src/assertions/mod.rs b/mithril-test-lab/mithril-end-to-end/src/assertions/mod.rs new file mode 100644 index 00000000000..8c71e254ae7 --- /dev/null +++ b/mithril-test-lab/mithril-end-to-end/src/assertions/mod.rs @@ -0,0 +1,376 @@ +use crate::{ + attempt, utils::AttemptResult, Aggregator, Client, ClientCommand, Devnet, + MithrilStakeDistributionCommand, SnapshotCommand, +}; +use mithril_common::{ + chain_observer::{CardanoCliChainObserver, ChainObserver}, + digesters::ImmutableFile, + entities::{Epoch, ProtocolParameters}, + messages::{ + CertificateMessage, EpochSettingsMessage, MithrilStakeDistributionListMessage, + MithrilStakeDistributionMessage, SnapshotMessage, + }, +}; +use reqwest::StatusCode; +use slog_scope::{info, warn}; +use std::{error::Error, path::Path, sync::Arc, time::Duration}; + +pub async fn wait_for_enough_immutable(db_directory: &Path) -> Result<(), String> { + info!("Waiting that enough immutable have been written in the devnet"); + + match attempt!(24, Duration::from_secs(5), { + match ImmutableFile::list_completed_in_dir(db_directory) + .map_err(|e| { + format!( + "Immutable file listing failed in dir `{}`: {}", + db_directory.display(), + e + ) + })? + .last() + { + Some(_) => Ok(Some(())), + None => Ok(None), + } + }) { + AttemptResult::Ok(_) => Ok(()), + AttemptResult::Err(error) => Err(error), + AttemptResult::Timeout() => Err(format!( + "Timeout exhausted for enough immutable to be written in `{}`", + db_directory.display() + )), + } +} + +pub async fn wait_for_epoch_settings( + aggregator_endpoint: &str, +) -> Result { + let url = format!("{aggregator_endpoint}/epoch-settings"); + info!("Waiting for the aggregator to expose epoch settings"); + + match attempt!(20, Duration::from_millis(1000), { + match reqwest::get(url.clone()).await { + Ok(response) => match response.status() { + StatusCode::OK => { + let epoch_settings = response + .json::() + .await + .map_err(|e| format!("Invalid EpochSettings body : {e}"))?; + info!("Aggregator ready"; "epoch_settings" => #?epoch_settings); + Ok(Some(epoch_settings)) + } + s if s.is_server_error() => { + warn!( + "Server error while waiting for the Aggregator, http code: {}", + s + ); + Ok(None) + } + _ => Ok(None), + }, + Err(_) => Ok(None), + } + }) { + AttemptResult::Ok(epoch_settings) => Ok(epoch_settings), + AttemptResult::Err(error) => Err(error), + AttemptResult::Timeout() => Err(format!( + "Timeout exhausted for aggregator to be up, no response from `{url}`" + )), + } +} + +pub async fn wait_for_target_epoch( + chain_observer: Arc, + target_epoch: Epoch, + wait_reason: String, +) -> Result<(), String> { + info!( + "Waiting for the cardano network to be at the target epoch: {}", wait_reason; + "target_epoch" => ?target_epoch + ); + + match attempt!(90, Duration::from_millis(1000), { + match chain_observer.get_current_epoch().await { + Ok(Some(epoch)) => { + if epoch >= target_epoch { + Ok(Some(())) + } else { + Ok(None) + } + } + Ok(None) => Ok(None), + Err(err) => Err(format!("Could not query current epoch: {err}")), + } + }) { + AttemptResult::Ok(_) => { + info!("Target epoch reached!"; "target_epoch" => ?target_epoch); + Ok(()) + } + AttemptResult::Err(error) => Err(error), + AttemptResult::Timeout() => { + Err("Timeout exhausted for target epoch to be reached".to_string()) + } + }?; + + Ok(()) +} + +pub async fn bootstrap_genesis_certificate(aggregator: &mut Aggregator) -> Result<(), String> { + info!("Bootstrap genesis certificate"); + + info!("> stopping aggregator"); + aggregator.stop().await?; + info!("> bootstrapping genesis using signers registered two epochs ago..."); + aggregator.bootstrap_genesis().await?; + info!("> done, restarting aggregator"); + aggregator.serve()?; + + Ok(()) +} + +pub async fn delegate_stakes_to_pools(devnet: &Devnet) -> Result<(), String> { + info!("Delegate stakes to the cardano pools"); + + devnet.delegate_stakes().await?; + + Ok(()) +} + +pub async fn update_protocol_parameters(aggregator: &mut Aggregator) -> Result<(), String> { + info!("Update protocol parameters"); + + info!("> stopping aggregator"); + aggregator.stop().await?; + let protocol_parameters_new = ProtocolParameters { + k: 150, + m: 210, + phi_f: 0.80, + }; + info!( + "> updating protocol parameters to {:?}...", + protocol_parameters_new + ); + aggregator.set_protocol_parameters(&protocol_parameters_new); + info!("> done, restarting aggregator"); + aggregator.serve()?; + + Ok(()) +} + +pub async fn assert_node_producing_mithril_stake_distribution( + aggregator_endpoint: &str, +) -> Result { + let url = format!("{aggregator_endpoint}/artifact/mithril-stake-distributions"); + info!("Waiting for the aggregator to produce a mithril stake distribution"); + + // todo: reduce the number of attempts if we can reduce the delay between two immutables + match attempt!(45, Duration::from_millis(2000), { + match reqwest::get(url.clone()).await { + Ok(response) => match response.status() { + StatusCode::OK => match response.json::().await.as_deref() { + Ok([stake_distribution, ..]) => Ok(Some(stake_distribution.hash.clone())), + Ok(&[]) => Ok(None), + Err(err) => Err(format!("Invalid mithril stake distribution body : {err}",)), + }, + s => Err(format!("Unexpected status code from Aggregator: {s}")), + }, + Err(err) => Err(format!("Request to `{url}` failed: {err}")), + } + }) { + AttemptResult::Ok(hash) => { + info!("Aggregator produced a mithril stake distribution"; "hash" => &hash); + Ok(hash) + } + AttemptResult::Err(error) => Err(error), + AttemptResult::Timeout() => Err(format!( + "Timeout exhausted assert_node_producing_mithril_stake_distribution, no response from `{url}`" + )), + } +} + +pub async fn assert_signer_is_signing_mithril_stake_distribution( + aggregator_endpoint: &str, + hash: &str, + expected_epoch_min: Epoch, +) -> Result { + let url = format!("{aggregator_endpoint}/artifact/mithril-stake-distribution/{hash}"); + info!( + "Asserting the aggregator is signing the mithril stake distribution message `{}` with an expected min epoch of `{}`", + hash, + expected_epoch_min + ); + + match attempt!(10, Duration::from_millis(1000), { + match reqwest::get(url.clone()).await { + Ok(response) => match response.status() { + StatusCode::OK => match response.json::().await { + Ok(stake_distribution) => match stake_distribution.epoch { + epoch if epoch >= expected_epoch_min => Ok(Some(stake_distribution)), + epoch => Err(format!( + "Minimum expected mithril stake distribution epoch not reached : {epoch} < {expected_epoch_min}" + )), + }, + Err(err) => Err(format!("Invalid mithril stake distribution body : {err}",)), + }, + StatusCode::NOT_FOUND => Ok(None), + s => Err(format!("Unexpected status code from Aggregator: {s}")), + }, + Err(err) => Err(format!("Request to `{url}` failed: {err}")), + } + }) { + AttemptResult::Ok(stake_distribution) => { + // todo: assert that the mithril stake distribution is really signed + info!("Signer signed a mithril stake distribution"; "certificate_hash" => &stake_distribution.certificate_hash); + Ok(stake_distribution.certificate_hash) + } + AttemptResult::Err(error) => Err(error), + AttemptResult::Timeout() => Err(format!( + "Timeout exhausted assert_signer_is_signing_mithril_stake_distribution, no response from `{url}`" + )), + } +} + +pub async fn assert_node_producing_snapshot(aggregator_endpoint: &str) -> Result { + let url = format!("{aggregator_endpoint}/artifact/snapshots"); + info!("Waiting for the aggregator to produce a snapshot"); + + // todo: reduce the number of attempts if we can reduce the delay between two immutables + match attempt!(45, Duration::from_millis(2000), { + match reqwest::get(url.clone()).await { + Ok(response) => match response.status() { + StatusCode::OK => match response.json::>().await.as_deref() { + Ok([snapshot, ..]) => Ok(Some(snapshot.digest.clone())), + Ok(&[]) => Ok(None), + Err(err) => Err(format!("Invalid snapshot body : {err}",)), + }, + s => Err(format!("Unexpected status code from Aggregator: {s}")), + }, + Err(err) => Err(format!("Request to `{url}` failed: {err}")), + } + }) { + AttemptResult::Ok(digest) => { + info!("Aggregator produced a snapshot"; "digest" => &digest); + Ok(digest) + } + AttemptResult::Err(error) => Err(error), + AttemptResult::Timeout() => Err(format!( + "Timeout exhausted assert_node_producing_snapshot, no response from `{url}`" + )), + } +} + +pub async fn assert_signer_is_signing_snapshot( + aggregator_endpoint: &str, + digest: &str, + expected_epoch_min: Epoch, +) -> Result { + let url = format!("{aggregator_endpoint}/artifact/snapshot/{digest}"); + info!( + "Asserting the aggregator is signing the snapshot message `{}` with an expected min epoch of `{}`", + digest, + expected_epoch_min + ); + + match attempt!(10, Duration::from_millis(1000), { + match reqwest::get(url.clone()).await { + Ok(response) => match response.status() { + StatusCode::OK => match response.json::().await { + Ok(snapshot) => match snapshot.beacon.epoch { + epoch if epoch >= expected_epoch_min => Ok(Some(snapshot)), + epoch => Err(format!( + "Minimum expected snapshot epoch not reached : {epoch} < {expected_epoch_min}" + )), + }, + Err(err) => Err(format!("Invalid snapshot body : {err}",)), + }, + StatusCode::NOT_FOUND => Ok(None), + s => Err(format!("Unexpected status code from Aggregator: {s}")), + }, + Err(err) => Err(format!("Request to `{url}` failed: {err}")), + } + }) { + AttemptResult::Ok(snapshot) => { + // todo: assert that the snapshot is really signed + info!("Signer signed a snapshot"; "certificate_hash" => &snapshot.certificate_hash); + Ok(snapshot.certificate_hash) + } + AttemptResult::Err(error) => Err(error), + AttemptResult::Timeout() => Err(format!( + "Timeout exhausted assert_signer_is_signing_snapshot, no response from `{url}`" + )), + } +} + +pub async fn assert_is_creating_certificate_with_enough_signers( + aggregator_endpoint: &str, + certificate_hash: &str, + total_signers_expected: usize, +) -> Result<(), String> { + let url = format!("{aggregator_endpoint}/certificate/{certificate_hash}"); + + match attempt!(10, Duration::from_millis(1000), { + match reqwest::get(url.clone()).await { + Ok(response) => match response.status() { + StatusCode::OK => match response.json::().await { + Ok(certificate) => Ok(Some(certificate)), + Err(err) => Err(format!("Invalid snapshot body : {err}",)), + }, + StatusCode::NOT_FOUND => Ok(None), + s => Err(format!("Unexpected status code from Aggregator: {s}")), + }, + Err(err) => Err(format!("Request to `{url}` failed: {err}")), + } + }) { + AttemptResult::Ok(certificate) => { + info!("Aggregator produced a certificate"; "certificate" => #?certificate); + if certificate.metadata.signers.len() == total_signers_expected { + info!( + "Certificate is signed by expected number of signers: {} >= {} ", + certificate.metadata.signers.len(), + total_signers_expected + ); + Ok(()) + } else { + Err(format!( + "Certificate is not signed by expected number of signers: {} < {} ", + certificate.metadata.signers.len(), + total_signers_expected + )) + } + } + AttemptResult::Err(error) => Err(error), + AttemptResult::Timeout() => Err(format!( + "Timeout exhausted assert_is_creating_certificate, no response from `{url}`" + )), + } +} + +pub async fn assert_client_can_verify_snapshot( + client: &mut Client, + digest: &str, +) -> Result<(), String> { + client + .run(ClientCommand::Snapshot(SnapshotCommand::Download { + digest: digest.to_string(), + })) + .await?; + info!("Client downloaded & restored the snapshot"; "digest" => &digest); + + Ok(()) +} + +pub async fn assert_client_can_verify_mithril_stake_distribution( + client: &mut Client, + hash: &str, +) -> Result<(), Box> { + client + .run(ClientCommand::MithrilStakeDistribution( + MithrilStakeDistributionCommand::Download { + hash: hash.to_owned(), + }, + )) + .await?; + info!("Client downloaded the Mithril stake distribution"; "hash" => &hash); + + Ok(()) +} diff --git a/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs b/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs index 73c86d047e0..81a2ec2f039 100644 --- a/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs +++ b/mithril-test-lab/mithril-end-to-end/src/end_to_end_spec.rs @@ -1,23 +1,7 @@ -use crate::utils::AttemptResult; -use crate::{ - attempt, Aggregator, Client, ClientCommand, Devnet, MithrilInfrastructure, - MithrilStakeDistributionCommand, SnapshotCommand, -}; -use mithril_common::{ - chain_observer::{CardanoCliChainObserver, ChainObserver}, - digesters::ImmutableFile, - entities::{Epoch, ProtocolParameters}, - messages::{ - CertificateMessage, EpochSettingsMessage, MithrilStakeDistributionListMessage, - MithrilStakeDistributionMessage, SnapshotMessage, - }, -}; -use reqwest::StatusCode; -use slog_scope::{info, warn}; +use crate::assertions; +use crate::MithrilInfrastructure; +use mithril_common::chain_observer::ChainObserver; use std::error::Error; -use std::path::Path; -use std::sync::Arc; -use std::time::Duration; pub struct Spec { infrastructure: MithrilInfrastructure, @@ -30,7 +14,8 @@ impl Spec { pub async fn run(&mut self) -> Result<(), Box> { let aggregator_endpoint = self.infrastructure.aggregator().endpoint(); - wait_for_enough_immutable(self.infrastructure.aggregator().db_directory()).await?; + assertions::wait_for_enough_immutable(self.infrastructure.aggregator().db_directory()) + .await?; let start_epoch = self .infrastructure .chain_observer() @@ -40,39 +25,39 @@ impl Spec { // Wait 3 epochs after start epoch for the aggregator to be able to bootstrap a genesis certificate let mut target_epoch = start_epoch + 3; - wait_for_target_epoch( + assertions::wait_for_target_epoch( self.infrastructure.chain_observer(), target_epoch, "minimal epoch for the aggregator to be able to bootstrap genesis certificate" .to_string(), ) .await?; - bootstrap_genesis_certificate(self.infrastructure.aggregator_mut()).await?; - wait_for_epoch_settings(&aggregator_endpoint).await?; + assertions::bootstrap_genesis_certificate(self.infrastructure.aggregator_mut()).await?; + assertions::wait_for_epoch_settings(&aggregator_endpoint).await?; // Wait 2 epochs before changing stake distribution, so that we use at least one original stake distribution target_epoch += 2; - wait_for_target_epoch( + assertions::wait_for_target_epoch( self.infrastructure.chain_observer(), target_epoch, "epoch after which the stake distribution will change".to_string(), ) .await?; - delegate_stakes_to_pools(self.infrastructure.devnet()).await?; + assertions::delegate_stakes_to_pools(self.infrastructure.devnet()).await?; // Wait 2 epochs before changing protocol parameters target_epoch += 2; - wait_for_target_epoch( + assertions::wait_for_target_epoch( self.infrastructure.chain_observer(), target_epoch, "epoch after which the protocol parameters will change".to_string(), ) .await?; - update_protocol_parameters(self.infrastructure.aggregator_mut()).await?; + assertions::update_protocol_parameters(self.infrastructure.aggregator_mut()).await?; // Wait 5 epochs after protocol parameters update, so that we make sure that we use new protocol parameters as well as new stake distribution a few times target_epoch += 5; - wait_for_target_epoch( + assertions::wait_for_target_epoch( self.infrastructure.chain_observer(), target_epoch, "epoch after which the certificate chain will be long enough to catch most common troubles with stake distribution and protocol parameters".to_string(), @@ -82,38 +67,45 @@ impl Spec { // Verify that mithril stake distribution artifacts are produced and signed correctly { let hash = - assert_node_producing_mithril_stake_distribution(&aggregator_endpoint).await?; - let certificate_hash = assert_signer_is_signing_mithril_stake_distribution( + assertions::assert_node_producing_mithril_stake_distribution(&aggregator_endpoint) + .await?; + let certificate_hash = assertions::assert_signer_is_signing_mithril_stake_distribution( &aggregator_endpoint, &hash, target_epoch - 2, ) .await?; - assert_is_creating_certificate_with_enough_signers( + assertions::assert_is_creating_certificate_with_enough_signers( &aggregator_endpoint, &certificate_hash, self.infrastructure.signers().len(), ) .await?; let mut client = self.infrastructure.build_client()?; - assert_client_can_verify_mithril_stake_distribution(&mut client, &hash).await?; + assertions::assert_client_can_verify_mithril_stake_distribution(&mut client, &hash) + .await?; } // Verify that snapshot artifacts are produced and signed correctly - let digest = assert_node_producing_snapshot(&aggregator_endpoint).await?; - let certificate_hash = - assert_signer_is_signing_snapshot(&aggregator_endpoint, &digest, target_epoch - 2) - .await?; + { + let digest = assertions::assert_node_producing_snapshot(&aggregator_endpoint).await?; + let certificate_hash = assertions::assert_signer_is_signing_snapshot( + &aggregator_endpoint, + &digest, + target_epoch - 2, + ) + .await?; - assert_is_creating_certificate_with_enough_signers( - &aggregator_endpoint, - &certificate_hash, - self.infrastructure.signers().len(), - ) - .await?; + assertions::assert_is_creating_certificate_with_enough_signers( + &aggregator_endpoint, + &certificate_hash, + self.infrastructure.signers().len(), + ) + .await?; - let mut client = self.infrastructure.build_client()?; - assert_client_can_verify_snapshot(&mut client, &digest).await?; + let mut client = self.infrastructure.build_client()?; + assertions::assert_client_can_verify_snapshot(&mut client, &digest).await?; + } Ok(()) } @@ -130,363 +122,3 @@ impl Spec { Ok(()) } } - -async fn wait_for_enough_immutable(db_directory: &Path) -> Result<(), String> { - info!("Waiting that enough immutable have been written in the devnet"); - - match attempt!(24, Duration::from_secs(5), { - match ImmutableFile::list_completed_in_dir(db_directory) - .map_err(|e| { - format!( - "Immutable file listing failed in dir `{}`: {}", - db_directory.display(), - e - ) - })? - .last() - { - Some(_) => Ok(Some(())), - None => Ok(None), - } - }) { - AttemptResult::Ok(_) => Ok(()), - AttemptResult::Err(error) => Err(error), - AttemptResult::Timeout() => Err(format!( - "Timeout exhausted for enough immutable to be written in `{}`", - db_directory.display() - )), - } -} - -async fn wait_for_epoch_settings( - aggregator_endpoint: &str, -) -> Result { - let url = format!("{aggregator_endpoint}/epoch-settings"); - info!("Waiting for the aggregator to expose epoch settings"); - - match attempt!(20, Duration::from_millis(1000), { - match reqwest::get(url.clone()).await { - Ok(response) => match response.status() { - StatusCode::OK => { - let epoch_settings = response - .json::() - .await - .map_err(|e| format!("Invalid EpochSettings body : {e}"))?; - info!("Aggregator ready"; "epoch_settings" => #?epoch_settings); - Ok(Some(epoch_settings)) - } - s if s.is_server_error() => { - warn!( - "Server error while waiting for the Aggregator, http code: {}", - s - ); - Ok(None) - } - _ => Ok(None), - }, - Err(_) => Ok(None), - } - }) { - AttemptResult::Ok(epoch_settings) => Ok(epoch_settings), - AttemptResult::Err(error) => Err(error), - AttemptResult::Timeout() => Err(format!( - "Timeout exhausted for aggregator to be up, no response from `{url}`" - )), - } -} - -async fn wait_for_target_epoch( - chain_observer: Arc, - target_epoch: Epoch, - wait_reason: String, -) -> Result<(), String> { - info!( - "Waiting for the cardano network to be at the target epoch: {}", wait_reason; - "target_epoch" => ?target_epoch - ); - - match attempt!(90, Duration::from_millis(1000), { - match chain_observer.get_current_epoch().await { - Ok(Some(epoch)) => { - if epoch >= target_epoch { - Ok(Some(())) - } else { - Ok(None) - } - } - Ok(None) => Ok(None), - Err(err) => Err(format!("Could not query current epoch: {err}")), - } - }) { - AttemptResult::Ok(_) => { - info!("Target epoch reached!"; "target_epoch" => ?target_epoch); - Ok(()) - } - AttemptResult::Err(error) => Err(error), - AttemptResult::Timeout() => { - Err("Timeout exhausted for target epoch to be reached".to_string()) - } - }?; - - Ok(()) -} - -async fn bootstrap_genesis_certificate(aggregator: &mut Aggregator) -> Result<(), String> { - info!("Bootstrap genesis certificate"); - - info!("> stopping aggregator"); - aggregator.stop().await?; - info!("> bootstrapping genesis using signers registered two epochs ago..."); - aggregator.bootstrap_genesis().await?; - info!("> done, restarting aggregator"); - aggregator.serve()?; - - Ok(()) -} - -async fn delegate_stakes_to_pools(devnet: &Devnet) -> Result<(), String> { - info!("Delegate stakes to the cardano pools"); - - devnet.delegate_stakes().await?; - - Ok(()) -} - -async fn update_protocol_parameters(aggregator: &mut Aggregator) -> Result<(), String> { - info!("Update protocol parameters"); - - info!("> stopping aggregator"); - aggregator.stop().await?; - let protocol_parameters_new = ProtocolParameters { - k: 150, - m: 210, - phi_f: 0.80, - }; - info!( - "> updating protocol parameters to {:?}...", - protocol_parameters_new - ); - aggregator.set_protocol_parameters(&protocol_parameters_new); - info!("> done, restarting aggregator"); - aggregator.serve()?; - - Ok(()) -} - -async fn assert_node_producing_mithril_stake_distribution( - aggregator_endpoint: &str, -) -> Result { - let url = format!("{aggregator_endpoint}/artifact/mithril-stake-distributions"); - info!("Waiting for the aggregator to produce a mithril stake distribution"); - - // todo: reduce the number of attempts if we can reduce the delay between two immutables - match attempt!(45, Duration::from_millis(2000), { - match reqwest::get(url.clone()).await { - Ok(response) => match response.status() { - StatusCode::OK => match response.json::().await.as_deref() { - Ok([stake_distribution, ..]) => Ok(Some(stake_distribution.hash.clone())), - Ok(&[]) => Ok(None), - Err(err) => Err(format!("Invalid mithril stake distribution body : {err}",)), - }, - s => Err(format!("Unexpected status code from Aggregator: {s}")), - }, - Err(err) => Err(format!("Request to `{url}` failed: {err}")), - } - }) { - AttemptResult::Ok(hash) => { - info!("Aggregator produced a mithril stake distribution"; "hash" => &hash); - Ok(hash) - } - AttemptResult::Err(error) => Err(error), - AttemptResult::Timeout() => Err(format!( - "Timeout exhausted assert_node_producing_mithril_stake_distribution, no response from `{url}`" - )), - } -} - -async fn assert_signer_is_signing_mithril_stake_distribution( - aggregator_endpoint: &str, - hash: &str, - expected_epoch_min: Epoch, -) -> Result { - let url = format!("{aggregator_endpoint}/artifact/mithril-stake-distribution/{hash}"); - info!( - "Asserting the aggregator is signing the mithril stake distribution message `{}` with an expected min epoch of `{}`", - hash, - expected_epoch_min - ); - - match attempt!(10, Duration::from_millis(1000), { - match reqwest::get(url.clone()).await { - Ok(response) => match response.status() { - StatusCode::OK => match response.json::().await { - Ok(stake_distribution) => match stake_distribution.epoch { - epoch if epoch >= expected_epoch_min => Ok(Some(stake_distribution)), - epoch => Err(format!( - "Minimum expected mithril stake distribution epoch not reached : {epoch} < {expected_epoch_min}" - )), - }, - Err(err) => Err(format!("Invalid mithril stake distribution body : {err}",)), - }, - StatusCode::NOT_FOUND => Ok(None), - s => Err(format!("Unexpected status code from Aggregator: {s}")), - }, - Err(err) => Err(format!("Request to `{url}` failed: {err}")), - } - }) { - AttemptResult::Ok(stake_distribution) => { - // todo: assert that the mithril stake distribution is really signed - info!("Signer signed a mithril stake distribution"; "certificate_hash" => &stake_distribution.certificate_hash); - Ok(stake_distribution.certificate_hash) - } - AttemptResult::Err(error) => Err(error), - AttemptResult::Timeout() => Err(format!( - "Timeout exhausted assert_signer_is_signing_mithril_stake_distribution, no response from `{url}`" - )), - } -} - -async fn assert_node_producing_snapshot(aggregator_endpoint: &str) -> Result { - let url = format!("{aggregator_endpoint}/artifact/snapshots"); - info!("Waiting for the aggregator to produce a snapshot"); - - // todo: reduce the number of attempts if we can reduce the delay between two immutables - match attempt!(45, Duration::from_millis(2000), { - match reqwest::get(url.clone()).await { - Ok(response) => match response.status() { - StatusCode::OK => match response.json::>().await.as_deref() { - Ok([snapshot, ..]) => Ok(Some(snapshot.digest.clone())), - Ok(&[]) => Ok(None), - Err(err) => Err(format!("Invalid snapshot body : {err}",)), - }, - s => Err(format!("Unexpected status code from Aggregator: {s}")), - }, - Err(err) => Err(format!("Request to `{url}` failed: {err}")), - } - }) { - AttemptResult::Ok(digest) => { - info!("Aggregator produced a snapshot"; "digest" => &digest); - Ok(digest) - } - AttemptResult::Err(error) => Err(error), - AttemptResult::Timeout() => Err(format!( - "Timeout exhausted assert_node_producing_snapshot, no response from `{url}`" - )), - } -} - -async fn assert_signer_is_signing_snapshot( - aggregator_endpoint: &str, - digest: &str, - expected_epoch_min: Epoch, -) -> Result { - let url = format!("{aggregator_endpoint}/artifact/snapshot/{digest}"); - info!( - "Asserting the aggregator is signing the snapshot message `{}` with an expected min epoch of `{}`", - digest, - expected_epoch_min - ); - - match attempt!(10, Duration::from_millis(1000), { - match reqwest::get(url.clone()).await { - Ok(response) => match response.status() { - StatusCode::OK => match response.json::().await { - Ok(snapshot) => match snapshot.beacon.epoch { - epoch if epoch >= expected_epoch_min => Ok(Some(snapshot)), - epoch => Err(format!( - "Minimum expected snapshot epoch not reached : {epoch} < {expected_epoch_min}" - )), - }, - Err(err) => Err(format!("Invalid snapshot body : {err}",)), - }, - StatusCode::NOT_FOUND => Ok(None), - s => Err(format!("Unexpected status code from Aggregator: {s}")), - }, - Err(err) => Err(format!("Request to `{url}` failed: {err}")), - } - }) { - AttemptResult::Ok(snapshot) => { - // todo: assert that the snapshot is really signed - info!("Signer signed a snapshot"; "certificate_hash" => &snapshot.certificate_hash); - Ok(snapshot.certificate_hash) - } - AttemptResult::Err(error) => Err(error), - AttemptResult::Timeout() => Err(format!( - "Timeout exhausted assert_signer_is_signing_snapshot, no response from `{url}`" - )), - } -} - -async fn assert_is_creating_certificate_with_enough_signers( - aggregator_endpoint: &str, - certificate_hash: &str, - total_signers_expected: usize, -) -> Result<(), String> { - let url = format!("{aggregator_endpoint}/certificate/{certificate_hash}"); - - match attempt!(10, Duration::from_millis(1000), { - match reqwest::get(url.clone()).await { - Ok(response) => match response.status() { - StatusCode::OK => match response.json::().await { - Ok(certificate) => Ok(Some(certificate)), - Err(err) => Err(format!("Invalid snapshot body : {err}",)), - }, - StatusCode::NOT_FOUND => Ok(None), - s => Err(format!("Unexpected status code from Aggregator: {s}")), - }, - Err(err) => Err(format!("Request to `{url}` failed: {err}")), - } - }) { - AttemptResult::Ok(certificate) => { - info!("Aggregator produced a certificate"; "certificate" => #?certificate); - if certificate.metadata.signers.len() == total_signers_expected { - info!( - "Certificate is signed by expected number of signers: {} >= {} ", - certificate.metadata.signers.len(), - total_signers_expected - ); - Ok(()) - } else { - Err(format!( - "Certificate is not signed by expected number of signers: {} < {} ", - certificate.metadata.signers.len(), - total_signers_expected - )) - } - } - AttemptResult::Err(error) => Err(error), - AttemptResult::Timeout() => Err(format!( - "Timeout exhausted assert_is_creating_certificate, no response from `{url}`" - )), - } -} - -async fn assert_client_can_verify_snapshot( - client: &mut Client, - digest: &str, -) -> Result<(), String> { - client - .run(ClientCommand::Snapshot(SnapshotCommand::Download { - digest: digest.to_string(), - })) - .await?; - info!("Client downloaded & restored the snapshot"; "digest" => &digest); - - Ok(()) -} - -async fn assert_client_can_verify_mithril_stake_distribution( - client: &mut Client, - hash: &str, -) -> Result<(), Box> { - client - .run(ClientCommand::MithrilStakeDistribution( - MithrilStakeDistributionCommand::Download { - hash: hash.to_owned(), - }, - )) - .await?; - info!("Client downloaded the Mithril stake distribution"; "hash" => &hash); - - Ok(()) -} diff --git a/mithril-test-lab/mithril-end-to-end/src/lib.rs b/mithril-test-lab/mithril-end-to-end/src/lib.rs index c285c79068a..891b6d619b6 100644 --- a/mithril-test-lab/mithril-end-to-end/src/lib.rs +++ b/mithril-test-lab/mithril-end-to-end/src/lib.rs @@ -1,8 +1,11 @@ +pub mod assertions; mod devnet; mod end_to_end_spec; mod mithril; +mod run_only; mod utils; pub use devnet::Devnet; pub use end_to_end_spec::Spec; pub use mithril::*; +pub use run_only::RunOnly; diff --git a/mithril-test-lab/mithril-end-to-end/src/main.rs b/mithril-test-lab/mithril-end-to-end/src/main.rs index b1f968af6fd..bd4104f1e36 100644 --- a/mithril-test-lab/mithril-end-to-end/src/main.rs +++ b/mithril-test-lab/mithril-end-to-end/src/main.rs @@ -1,6 +1,6 @@ use clap::Parser; -use mithril_end_to_end::Spec; use mithril_end_to_end::{Devnet, MithrilInfrastructure}; +use mithril_end_to_end::{RunOnly, Spec}; use slog::{Drain, Logger}; use slog_scope::error; use std::error::Error; @@ -49,6 +49,10 @@ pub struct Args { /// Mithril era to run #[clap(long, default_value = "thales")] mithril_era: String, + + /// Enable run only mode + #[clap(long)] + run_only: bool, } #[tokio::main] @@ -67,6 +71,7 @@ async fn main() -> Result<(), Box> { work_dir.canonicalize().unwrap() } }; + let run_only_mode = args.run_only; let devnet = Devnet::bootstrap( args.devnet_scripts_directory, @@ -84,22 +89,44 @@ async fn main() -> Result<(), Box> { &work_dir, &args.bin_directory, &args.mithril_era, + run_only_mode, ) .await?; - let mut spec = Spec::new(infrastructure); - - match spec.run().await { - Ok(_) => { - devnet.stop().await?; - Ok(()) + match run_only_mode { + true => { + let mut run_only = RunOnly::new(infrastructure); + + match run_only.run().await { + Ok(_) => { + devnet.stop().await?; + Ok(()) + } + Err(error) => { + let has_written_logs = run_only.tail_logs(20).await; + error!("Mithril End to End test in run-only mode failed: {}", error); + devnet.stop().await?; + has_written_logs?; + Err(error) + } + } } - Err(error) => { - let has_written_logs = spec.tail_logs(20).await; - error!("Mithril End to End test failed: {}", error); - devnet.stop().await?; - has_written_logs?; - Err(error) + false => { + let mut spec = Spec::new(infrastructure); + + match spec.run().await { + Ok(_) => { + devnet.stop().await?; + Ok(()) + } + Err(error) => { + let has_written_logs = spec.tail_logs(20).await; + error!("Mithril End to End test failed: {}", error); + devnet.stop().await?; + has_written_logs?; + Err(error) + } + } } } } diff --git a/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs b/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs index de23edc75a2..72a836e7b1a 100644 --- a/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs +++ b/mithril-test-lab/mithril-end-to-end/src/mithril/infrastructure.rs @@ -13,6 +13,7 @@ pub struct MithrilInfrastructure { aggregator: Aggregator, signers: Vec, cardano_chain_observer: Arc, + run_only_mode: bool, } impl MithrilInfrastructure { @@ -22,6 +23,7 @@ impl MithrilInfrastructure { work_dir: &Path, bin_dir: &Path, mithril_era: &str, + run_only_mode: bool, ) -> Result { devnet.run().await?; let devnet_topology = devnet.topology(); @@ -81,6 +83,7 @@ impl MithrilInfrastructure { aggregator, signers, cardano_chain_observer, + run_only_mode, }) } @@ -111,4 +114,8 @@ impl MithrilInfrastructure { pub fn build_client(&self) -> Result { Client::new(self.aggregator.endpoint(), &self.work_dir, &self.bin_dir) } + + pub fn run_only_mode(&self) -> bool { + self.run_only_mode + } } diff --git a/mithril-test-lab/mithril-end-to-end/src/run_only.rs b/mithril-test-lab/mithril-end-to-end/src/run_only.rs new file mode 100644 index 00000000000..4262c035a11 --- /dev/null +++ b/mithril-test-lab/mithril-end-to-end/src/run_only.rs @@ -0,0 +1,87 @@ +use crate::assertions; +use crate::MithrilInfrastructure; +use mithril_common::chain_observer::ChainObserver; +use slog_scope::info; +use std::{error::Error, thread::sleep, time::Duration}; + +pub struct RunOnly { + infrastructure: MithrilInfrastructure, +} + +impl RunOnly { + pub fn new(infrastructure: MithrilInfrastructure) -> Self { + Self { infrastructure } + } + + pub async fn run(&mut self) -> Result<(), Box> { + let aggregator_endpoint = self.infrastructure.aggregator().endpoint(); + assertions::wait_for_enough_immutable(self.infrastructure.aggregator().db_directory()) + .await?; + let start_epoch = self + .infrastructure + .chain_observer() + .get_current_epoch() + .await? + .unwrap_or_default(); + + // Wait 3 epochs after start epoch for the aggregator to be able to bootstrap a genesis certificate + let mut target_epoch = start_epoch + 3; + assertions::wait_for_target_epoch( + self.infrastructure.chain_observer(), + target_epoch, + "minimal epoch for the aggregator to be able to bootstrap genesis certificate" + .to_string(), + ) + .await?; + assertions::bootstrap_genesis_certificate(self.infrastructure.aggregator_mut()).await?; + assertions::wait_for_epoch_settings(&aggregator_endpoint).await?; + + // Wait 2 epochs before changing stake distribution, so that we use at least one original stake distribution + target_epoch += 2; + assertions::wait_for_target_epoch( + self.infrastructure.chain_observer(), + target_epoch, + "epoch after which the stake distribution will change".to_string(), + ) + .await?; + assertions::delegate_stakes_to_pools(self.infrastructure.devnet()).await?; + + // Wait 2 epochs before changing protocol parameters + target_epoch += 2; + assertions::wait_for_target_epoch( + self.infrastructure.chain_observer(), + target_epoch, + "epoch after which the protocol parameters will change".to_string(), + ) + .await?; + assertions::update_protocol_parameters(self.infrastructure.aggregator_mut()).await?; + + // Wait 5 epochs after protocol parameters update, so that we make sure that we use new protocol parameters as well as new stake distribution a few times + target_epoch += 5; + assertions::wait_for_target_epoch( + self.infrastructure.chain_observer(), + target_epoch, + "epoch after which the certificate chain will be long enough to catch most common troubles with stake distribution and protocol parameters".to_string(), + ) + .await?; + + loop { + info!("Mithril end to end is running and will remain active until manually stopped..."); + sleep(Duration::from_secs(5)); + } + + Ok(()) + } + + pub async fn tail_logs(&self, number_of_line: u64) -> Result<(), String> { + self.infrastructure + .aggregator() + .tail_logs(number_of_line) + .await?; + for signer in self.infrastructure.signers() { + signer.tail_logs(number_of_line).await?; + } + + Ok(()) + } +}