From 2db4e1352bb281294d8673b1b0019463e299a7af Mon Sep 17 00:00:00 2001 From: FITAHIANA Nomeniavo joe <24nomeniavo@gmail.com> Date: Fri, 23 Aug 2024 22:35:30 +0300 Subject: [PATCH] feat: Add caching to Secrets struct to improve performance (#813) #### Migration notes - [ ] The change comes with new or modified tests - [ ] Hard-to-understand functions have explanatory comments - [ ] End-user documentation is updated to reflect the change --- meta-cli/src/cli/deploy.rs | 8 +-- meta-cli/src/deploy/actors/task/deploy.rs | 11 ++-- meta-cli/src/secrets.rs | 79 ++++++++++------------- 3 files changed, 43 insertions(+), 55 deletions(-) diff --git a/meta-cli/src/cli/deploy.rs b/meta-cli/src/cli/deploy.rs index 309028d321..df57de5105 100644 --- a/meta-cli/src/cli/deploy.rs +++ b/meta-cli/src/cli/deploy.rs @@ -114,7 +114,7 @@ impl Deploy { let options = deploy.options.clone(); let node_config = config.node(&deploy.node, &deploy.target); - let secrets = Secrets::load_from_node_config(&node_config); + let secrets = Secrets::load_from_node_config(&node_config, dir.to_path_buf()); let node = node_config .build(&dir) .await @@ -209,8 +209,7 @@ mod default_mode { let action_generator = DeployActionGenerator::new( deploy.node.into(), - // TODO no hydrate here - secrets.hydrate(deploy.base_dir.clone()).await?.into(), + secrets.into(), deploy.config.dir().unwrap_or_log().into(), deploy.base_dir.clone(), deploy @@ -293,8 +292,7 @@ mod watch_mode { let action_generator = DeployActionGenerator::new( deploy.node.into(), - // TODO no hydrate here - secrets.hydrate(deploy.base_dir.clone()).await?.into(), + secrets.into(), deploy.config.dir().unwrap_or_log().into(), deploy.base_dir.clone(), deploy diff --git a/meta-cli/src/deploy/actors/task/deploy.rs b/meta-cli/src/deploy/actors/task/deploy.rs index bfcda7170b..0809e8aab1 100644 --- a/meta-cli/src/deploy/actors/task/deploy.rs +++ b/meta-cli/src/deploy/actors/task/deploy.rs @@ -304,7 +304,7 @@ impl TaskAction for DeployAction { Ok(serde_json::to_value(deploy_target)?) } - RpcCall::GetDeployData { typegraph } => Ok(self.get_deploy_data(typegraph)), + RpcCall::GetDeployData { typegraph } => Ok(self.get_deploy_data(typegraph).await?), } } } @@ -321,7 +321,7 @@ impl MigrationAction { } impl DeployActionInner { - fn get_deploy_data(&self, typegraph: &str) -> serde_json::Value { + async fn get_deploy_data(&self, typegraph: &str) -> Result { let default_action = &self.shared_config.default_migration_action; let actions = self .task_options @@ -339,11 +339,10 @@ impl DeployActionInner { }) .collect::>(); - // TODO hydrate secrets here + cache - serde_json::json!({ - "secrets": self.secrets.get(typegraph), + Ok(serde_json::json!({ + "secrets": self.secrets.get(typegraph).await?, "defaultMigrationAction": default_action, "migrationActions": actions - }) + })) } } diff --git a/meta-cli/src/secrets.rs b/meta-cli/src/secrets.rs index 8c505b42db..7583f16c2d 100644 --- a/meta-cli/src/secrets.rs +++ b/meta-cli/src/secrets.rs @@ -1,24 +1,24 @@ // Copyright Metatype OÜ, licensed under the Mozilla Public License Version 2.0. // SPDX-License-Identifier: MPL-2.0 -use crate::interlude::*; +use tokio::sync::Mutex; use crate::config::NodeConfig; +use crate::interlude::*; -#[derive(Debug, Clone)] -pub struct Raw; -#[derive(Debug, Clone, Default)] -pub struct Hydrated; +lazy_static::lazy_static! { + static ref CACHED_SECRETS: Mutex>> = Mutex::new(HashMap::new()); +} // tg_name -> key -> value #[derive(Debug, Clone, Default)] -pub struct Secrets { +pub struct Secrets { by_typegraph: HashMap>, overrides: HashMap, - _type_state: std::marker::PhantomData, + path: PathBuf, } -pub type RawSecrets = Secrets; +pub type RawSecrets = Secrets; struct SecretOverride<'a> { tg_name: Option<&'a str>, @@ -54,12 +54,12 @@ impl<'a> SecretOverride<'a> { } } -impl Secrets { - pub fn load_from_node_config(node_config: &NodeConfig) -> Secrets { +impl Secrets { + pub fn load_from_node_config(node_config: &NodeConfig, path: PathBuf) -> Secrets { Secrets { by_typegraph: node_config.secrets.clone(), overrides: HashMap::new(), - _type_state: std::marker::PhantomData, + path, } } @@ -87,43 +87,34 @@ impl Secrets { Ok(()) } - #[tracing::instrument(err)] - pub async fn hydrate(self, dir: Arc) -> Result> { - let by_typegraph: HashMap = - futures::future::join_all(self.by_typegraph.into_iter().map(|(tg_name, secrets)| { - let dir = dir.clone(); - async move { - let secrets = lade_sdk::hydrate(secrets, dir.clone().to_path_buf()) - .await - .map_err(anyhow_to_eyre!()) - .with_context(|| format!("error hydrating secrets for {tg_name}"))?; - Ok((tg_name, secrets)) - } - })) - .await - .into_iter() - .collect::>()?; - - let overrides = lade_sdk::hydrate(self.overrides, dir.to_path_buf()) - .await - .map_err(|err| ferr!("error hydrating secrets overrides: {err}"))?; - - Ok(Secrets { - by_typegraph, - overrides, - _type_state: std::marker::PhantomData, - }) - } -} + pub async fn get(&self, tg_name: &str) -> Result> { + let mut cache = CACHED_SECRETS.lock().await; + if let Some(cached_result) = cache.get(tg_name) { + return Ok(cached_result.clone()); + } -impl Secrets { - pub fn get(&self, tg_name: &str) -> HashMap { let mut secrets = self.by_typegraph.get(tg_name).cloned().unwrap_or_default(); - for (key, value) in &self.overrides { - secrets.insert(key.clone(), value.clone()); + lade_sdk::hydrate(secrets.clone(), self.path.clone()) + .await + .map_err(anyhow_to_eyre!()) + .with_context(|| format!("error hydrating secrets for {tg_name}"))?; + + if !cache.contains_key("overrides") { + let hydrated_overrides = lade_sdk::hydrate(self.overrides.clone(), self.path.clone()) + .await + .map_err(|err| ferr!("error hydrating secrets overrides: {err}"))?; + cache.insert("overrides".to_string(), hydrated_overrides); } - secrets + if let Some(overrides) = cache.get("overrides") { + for (key, value) in overrides { + secrets.insert(key.clone(), value.clone()); + } + } + + cache.insert(tg_name.to_string(), secrets.clone()); + + Ok(secrets) } }