From 30aa1fcc066af52d503991f4752dcd21f33c00cb Mon Sep 17 00:00:00 2001 From: itowlson Date: Mon, 9 Sep 2024 15:26:41 +1200 Subject: [PATCH] Load and compose components once at start of validation Signed-off-by: itowlson --- crates/environments/src/lib.rs | 52 ++++++++++----------- crates/environments/src/loader.rs | 77 ++++++++++++++++++++++--------- 2 files changed, 80 insertions(+), 49 deletions(-) diff --git a/crates/environments/src/lib.rs b/crates/environments/src/lib.rs index 2578c17583..606c5ca939 100644 --- a/crates/environments/src/lib.rs +++ b/crates/environments/src/lib.rs @@ -5,15 +5,14 @@ mod loader; use environment_definition::{TargetEnvironment, TargetWorld, TriggerType}; pub use loader::ResolutionContext; -use loader::{component_source, ComponentSourceLoader, ComponentToValidate}; +use loader::{load_and_resolve_all, ComponentToValidate}; pub async fn validate_application_against_environment_ids( env_ids: impl Iterator, app: &spin_manifest::schema::v2::AppManifest, resolution_context: &ResolutionContext, ) -> anyhow::Result<()> { - let envs = futures::future::join_all(env_ids.map(resolve_environment_id)).await; - let envs: Vec<_> = envs.into_iter().collect::>()?; + let envs = join_all_result(env_ids.map(resolve_environment_id)).await?; validate_application_against_environments(&envs, app, resolution_context).await } @@ -54,6 +53,8 @@ pub async fn validate_application_against_environments( app: &spin_manifest::schema::v2::AppManifest, resolution_context: &ResolutionContext, ) -> anyhow::Result<()> { + use futures::FutureExt; + for trigger_type in app.triggers.keys() { if let Some(env) = envs .iter() @@ -66,26 +67,15 @@ pub async fn validate_application_against_environments( } } - let components_by_trigger_type = app - .triggers - .iter() - .map(|(ty, ts)| { - ts.iter() - .map(|t| component_source(app, t)) - .collect::, _>>() - .map(|css| (ty, css)) - }) - .collect::, _>>()?; + let components_by_trigger_type_futs = app.triggers.iter().map(|(ty, ts)| { + load_and_resolve_all(app, ts, resolution_context) + .map(|css| css.map(|css| (ty.to_owned(), css))) + }); + let components_by_trigger_type = join_all_result(components_by_trigger_type_futs).await?; for (trigger_type, component) in components_by_trigger_type { for component in &component { - validate_component_against_environments( - envs, - trigger_type, - component, - resolution_context, - ) - .await?; + validate_component_against_environments(envs, &trigger_type, component).await?; } } @@ -96,7 +86,6 @@ async fn validate_component_against_environments( envs: &[TargetEnvironment], trigger_type: &TriggerType, component: &ComponentToValidate<'_>, - resolution_context: &ResolutionContext, ) -> anyhow::Result<()> { let worlds = envs .iter() @@ -110,21 +99,18 @@ async fn validate_component_against_environments( .map(|w| (e.name.as_str(), w)) }) .collect::, _>>()?; - validate_component_against_worlds(worlds.into_iter(), component, resolution_context).await?; + validate_component_against_worlds(worlds.into_iter(), component).await?; Ok(()) } async fn validate_component_against_worlds( target_worlds: impl Iterator, component: &ComponentToValidate<'_>, - resolution_context: &ResolutionContext, ) -> anyhow::Result<()> { - let loader = ComponentSourceLoader::new(resolution_context.wasm_loader()); - let wasm_bytes = spin_compose::compose(&loader, component).await?; + let wasm_bytes = component.wasm_bytes(); for (env_name, target_world) in target_worlds { - validate_wasm_against_any_world(env_name, target_world, component, wasm_bytes.as_ref()) - .await?; + validate_wasm_against_any_world(env_name, target_world, component, wasm_bytes).await?; } tracing::info!( @@ -223,3 +209,15 @@ async fn validate_wasm_against_world( }, } } + +/// Equivalent to futures::future::join_all, but specialised for iterators of +/// fallible futures. It returns a Result> instead of a Vec> - +/// this just moves the transposition boilerplate out of the main flow. +async fn join_all_result(iter: I) -> anyhow::Result> +where + I: IntoIterator, + I::Item: std::future::Future>, +{ + let vec_result = futures::future::join_all(iter).await; + vec_result.into_iter().collect() +} diff --git a/crates/environments/src/loader.rs b/crates/environments/src/loader.rs index a33a6dc0ea..0cd229bcd7 100644 --- a/crates/environments/src/loader.rs +++ b/crates/environments/src/loader.rs @@ -4,6 +4,12 @@ use anyhow::anyhow; use spin_common::ui::quoted_path; pub(crate) struct ComponentToValidate<'a> { + id: &'a str, + source_description: String, + wasm: Vec, +} + +struct ComponentSource<'a> { id: &'a str, source: &'a spin_manifest::schema::v2::ComponentSource, dependencies: WrappedComponentDependencies, @@ -14,22 +20,30 @@ impl<'a> ComponentToValidate<'a> { self.id } - pub fn source_description(&self) -> String { - match self.source { - spin_manifest::schema::v2::ComponentSource::Local(path) => { - format!("file {}", quoted_path(path)) - } - spin_manifest::schema::v2::ComponentSource::Remote { url, .. } => format!("URL {url}"), - spin_manifest::schema::v2::ComponentSource::Registry { package, .. } => { - format!("package {package}") - } - } + pub fn source_description(&self) -> &str { + &self.source_description + } + + pub fn wasm_bytes(&self) -> &[u8] { + &self.wasm } } -pub fn component_source<'a>( +pub async fn load_and_resolve_all<'a>( + app: &'a spin_manifest::schema::v2::AppManifest, + triggers: &'a [spin_manifest::schema::v2::Trigger], + resolution_context: &'a ResolutionContext, +) -> anyhow::Result>> { + let component_futures = triggers + .iter() + .map(|t| load_and_resolve_one(app, t, resolution_context)); + crate::join_all_result(component_futures).await +} + +async fn load_and_resolve_one<'a>( app: &'a spin_manifest::schema::v2::AppManifest, trigger: &'a spin_manifest::schema::v2::Trigger, + resolution_context: &'a ResolutionContext, ) -> anyhow::Result> { let component_spec = trigger .component @@ -50,10 +64,21 @@ pub fn component_source<'a>( (id, &component.source, &component.dependencies) } }; - Ok(ComponentToValidate { + + let component = ComponentSource { id, source, dependencies: WrappedComponentDependencies::new(dependencies), + }; + + let loader = ComponentSourceLoader::new(resolution_context.wasm_loader()); + + let wasm = spin_compose::compose(&loader, &component).await?; + + Ok(ComponentToValidate { + id, + source_description: source_description(component.source), + wasm, }) } @@ -68,33 +93,29 @@ impl ResolutionContext { Ok(Self { wasm_loader }) } - pub(crate) fn wasm_loader(&self) -> &spin_loader::WasmLoader { + fn wasm_loader(&self) -> &spin_loader::WasmLoader { &self.wasm_loader } } -pub(crate) struct ComponentSourceLoader<'a> { +struct ComponentSourceLoader<'a> { wasm_loader: &'a spin_loader::WasmLoader, - _phantom: std::marker::PhantomData<&'a usize>, } impl<'a> ComponentSourceLoader<'a> { pub fn new(wasm_loader: &'a spin_loader::WasmLoader) -> Self { - Self { - wasm_loader, - _phantom: std::marker::PhantomData, - } + Self { wasm_loader } } } #[async_trait::async_trait] impl<'a> spin_compose::ComponentSourceLoader for ComponentSourceLoader<'a> { - type Component = ComponentToValidate<'a>; + type Component = ComponentSource<'a>; type Dependency = WrappedComponentDependency; async fn load_component_source(&self, source: &Self::Component) -> anyhow::Result> { let path = self .wasm_loader - .load_component_source(source.id(), source.source) + .load_component_source(source.id, source.source) .await?; let bytes = tokio::fs::read(&path).await?; let component = spin_componentize::componentize_if_necessary(&bytes)?; @@ -144,7 +165,7 @@ impl WrappedComponentDependencies { } #[async_trait::async_trait] -impl<'a> spin_compose::ComponentLike for ComponentToValidate<'a> { +impl<'a> spin_compose::ComponentLike for ComponentSource<'a> { type Dependency = WrappedComponentDependency; fn dependencies( @@ -176,3 +197,15 @@ impl spin_compose::DependencyLike for WrappedComponentDependency { } } } + +fn source_description(source: &spin_manifest::schema::v2::ComponentSource) -> String { + match source { + spin_manifest::schema::v2::ComponentSource::Local(path) => { + format!("file {}", quoted_path(path)) + } + spin_manifest::schema::v2::ComponentSource::Remote { url, .. } => format!("URL {url}"), + spin_manifest::schema::v2::ComponentSource::Registry { package, .. } => { + format!("package {package}") + } + } +}