From 75c10cd383b492a337be93e3a563968c85329f1d Mon Sep 17 00:00:00 2001 From: David Alsh Date: Wed, 22 Jan 2025 05:22:13 +0000 Subject: [PATCH 1/2] using buffers to transfer AssetGraph to nodejs --- crates/atlaspack_core/src/asset_graph/mod.rs | 2 - .../src/asset_graph/serialize_asset_graph.rs | 172 ------------------ .../node-bindings/src/atlaspack/atlaspack.rs | 12 +- crates/node-bindings/src/atlaspack/mod.rs | 1 + .../src/atlaspack/serialize_asset_graph.rs | 66 +++++++ .../src/requests/AssetGraphRequestRust.js | 2 +- 6 files changed, 70 insertions(+), 185 deletions(-) delete mode 100644 crates/atlaspack_core/src/asset_graph/serialize_asset_graph.rs create mode 100644 crates/node-bindings/src/atlaspack/serialize_asset_graph.rs diff --git a/crates/atlaspack_core/src/asset_graph/mod.rs b/crates/atlaspack_core/src/asset_graph/mod.rs index 222511364..129a61a7f 100644 --- a/crates/atlaspack_core/src/asset_graph/mod.rs +++ b/crates/atlaspack_core/src/asset_graph/mod.rs @@ -1,8 +1,6 @@ #[allow(clippy::module_inception)] mod asset_graph; mod propagate_requested_symbols; -mod serialize_asset_graph; pub use self::asset_graph::*; pub use self::propagate_requested_symbols::*; -pub use self::serialize_asset_graph::*; diff --git a/crates/atlaspack_core/src/asset_graph/serialize_asset_graph.rs b/crates/atlaspack_core/src/asset_graph/serialize_asset_graph.rs deleted file mode 100644 index 2c4e13f85..000000000 --- a/crates/atlaspack_core/src/asset_graph/serialize_asset_graph.rs +++ /dev/null @@ -1,172 +0,0 @@ -use serde::Serialize; - -use crate::types::{Asset, Dependency}; - -use super::{AssetGraph, AssetGraphNode, DependencyState}; - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct SerializedDependency { - id: String, - dependency: Dependency, -} - -#[derive(Debug, Serialize)] -#[serde(tag = "type", rename_all = "camelCase")] -enum SerializedAssetGraphNode { - Root, - Entry, - Asset { - value: Asset, - }, - Dependency { - value: SerializedDependency, - has_deferred: bool, - }, -} - -pub fn serialize_asset_graph( - asset_graph: &AssetGraph, - max_str_len: usize, -) -> serde_json::Result> { - let mut nodes: Vec = Vec::new(); - let mut curr_node = String::default(); - - for node in asset_graph.nodes() { - let serialized_node = match node { - AssetGraphNode::Root => SerializedAssetGraphNode::Root, - AssetGraphNode::Entry => SerializedAssetGraphNode::Entry, - AssetGraphNode::Asset(asset_node) => SerializedAssetGraphNode::Asset { - value: asset_node.asset.clone(), - }, - AssetGraphNode::Dependency(dependency_node) => SerializedAssetGraphNode::Dependency { - value: SerializedDependency { - id: dependency_node.dependency.id(), - dependency: dependency_node.dependency.as_ref().clone(), - }, - has_deferred: dependency_node.state == DependencyState::Deferred, - }, - }; - - let str = serde_json::to_string(&serialized_node)?; - if curr_node.len() + str.len() < (max_str_len - 3) { - if !curr_node.is_empty() { - curr_node.push(','); - } - curr_node.push_str(&str); - } else { - // Add the existing node now as it has reached the max JavaScript string size - nodes.push(format!("[{curr_node}]")); - curr_node = str; - } - } - - // Add the current node if it did not overflow in size - if curr_node.len() < (max_str_len - 3) { - nodes.push(format!("[{curr_node}]")); - } - - Ok(nodes) -} - -#[cfg(test)] -mod tests { - use std::path::PathBuf; - - use serde_json::{json, Value}; - - use super::*; - - #[test] - fn serialize_nodes_handles_max_size() -> anyhow::Result<()> { - let mut graph = AssetGraph::new(); - - let entry = graph.add_entry_dependency(Dependency { - specifier: String::from("entry"), - ..Dependency::default() - }); - - let entry_asset = graph.add_asset(Asset { - file_path: PathBuf::from("entry"), - ..Asset::default() - }); - - graph.add_edge(&entry, &entry_asset); - - for i in 1..100 { - let node_index = graph.add_dependency(Dependency { - specifier: format!("dependency-{}", i), - ..Dependency::default() - }); - graph.add_edge(&entry_asset, &node_index); - } - - let max_str_len = 10000; - let nodes = serialize_asset_graph(&graph, max_str_len)?; - - assert_eq!(nodes.len(), 7); - - // Assert each string is less than the max size - for node in nodes.iter() { - assert!(node.len() < max_str_len); - } - - // Assert all the nodes are included and in the correct order - let first_entry = serde_json::from_str::(&nodes[0])?; - let first_entry = first_entry.as_array().unwrap(); - - assert_eq!(get_type(&first_entry[0]), json!("root")); - assert_eq!(get_dependency(&first_entry[1]), Some(json!("entry"))); - assert_eq!(get_asset(&first_entry[2]), Some(json!("entry"))); - - for i in 1..first_entry.len() - 2 { - assert_eq!( - get_dependency(&first_entry[i + 2]), - Some(json!(format!("dependency-{}", i))) - ); - } - - let mut specifier = first_entry.len() - 2; - for node in nodes[1..].iter() { - let entry = serde_json::from_str::(node)?; - let entry = entry.as_array().unwrap(); - - for value in entry { - assert_eq!( - get_dependency(value), - Some(json!(format!("dependency-{}", specifier))) - ); - - specifier += 1; - } - } - - Ok(()) - } - - fn get_type(node: &Value) -> Value { - node.get("type").unwrap().to_owned() - } - - fn get_dependency(value: &Value) -> Option { - assert_eq!(get_type(value), json!("dependency")); - - value - .get("value") - .unwrap() - .get("dependency") - .unwrap() - .get("specifier") - .map(|s| s.to_owned()) - } - - fn get_asset(value: &Value) -> Option { - assert_eq!(get_type(value), json!("asset")); - - value - .get("value") - .unwrap() - .get("filePath") - .map(|s| s.to_owned()) - } -} diff --git a/crates/node-bindings/src/atlaspack/atlaspack.rs b/crates/node-bindings/src/atlaspack/atlaspack.rs index 6ed7f9d29..6d38f2b4f 100644 --- a/crates/node-bindings/src/atlaspack/atlaspack.rs +++ b/crates/node-bindings/src/atlaspack/atlaspack.rs @@ -19,7 +19,6 @@ use atlaspack::rpc::nodejs::NodejsRpcFactory; use atlaspack::rpc::nodejs::NodejsWorker; use atlaspack::rpc::RpcFactoryRef; use atlaspack::Atlaspack; -use atlaspack_core::asset_graph::serialize_asset_graph; use atlaspack_core::types::AtlaspackOptions; use atlaspack_napi_helpers::JsTransferable; use atlaspack_package_manager::PackageManagerRef; @@ -27,6 +26,7 @@ use atlaspack_package_manager::PackageManagerRef; use super::file_system_napi::FileSystemNapi; use super::napi_result::NapiAtlaspackResult; use super::package_manager_napi::PackageManagerNapi; +use super::serialize_asset_graph::serialize_asset_graph; #[napi(object)] pub struct AtlaspackNapiBuildOptions { @@ -148,15 +148,7 @@ impl AtlaspackNapi { // not supplied as JavaScript Error types. The JavaScript layer needs to handle conversions deferred.resolve(move |env| match result { Ok(asset_graph) => { - let mut js_object = env.create_object()?; - - js_object.set_named_property("edges", env.to_js_value(&asset_graph.edges())?)?; - js_object.set_named_property( - "nodes", - serialize_asset_graph(&asset_graph, MAX_STRING_LENGTH)?, - )?; - - NapiAtlaspackResult::ok(&env, js_object) + NapiAtlaspackResult::ok(&env, serialize_asset_graph(&env, &asset_graph)?) } Err(error) => { let js_object = env.to_js_value(&AtlaspackError::from(&error))?; diff --git a/crates/node-bindings/src/atlaspack/mod.rs b/crates/node-bindings/src/atlaspack/mod.rs index 549c9d9aa..398b5ebf1 100644 --- a/crates/node-bindings/src/atlaspack/mod.rs +++ b/crates/node-bindings/src/atlaspack/mod.rs @@ -8,4 +8,5 @@ pub mod file_system_napi; pub mod monitoring; pub mod napi_result; pub mod package_manager_napi; +pub mod serialize_asset_graph; pub mod worker; diff --git a/crates/node-bindings/src/atlaspack/serialize_asset_graph.rs b/crates/node-bindings/src/atlaspack/serialize_asset_graph.rs new file mode 100644 index 000000000..ba08c470b --- /dev/null +++ b/crates/node-bindings/src/atlaspack/serialize_asset_graph.rs @@ -0,0 +1,66 @@ +use napi::{Env, JsObject}; +use serde::Serialize; + +use atlaspack_core::asset_graph::{AssetGraph, AssetGraphNode, DependencyState}; +use atlaspack_core::types::{Asset, Dependency}; + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SerializedDependency { + id: String, + dependency: Dependency, +} + +#[derive(Debug, Serialize)] +#[serde(tag = "type", rename_all = "camelCase")] +enum SerializedAssetGraphNode { + Root, + Entry, + Asset { + value: Asset, + }, + Dependency { + value: SerializedDependency, + has_deferred: bool, + }, +} + +/// Returns +/// ```typescript +/// type SerializedAssetGraph = { +/// edges: Array, +/// nodes: Array, +/// } +/// ``` +pub fn serialize_asset_graph(env: &Env, asset_graph: &AssetGraph) -> anyhow::Result { + let mut js_nodes = env.create_empty_array()?; + + for (i, node) in asset_graph.nodes().enumerate() { + let serialized_node = match node { + AssetGraphNode::Root => SerializedAssetGraphNode::Root, + AssetGraphNode::Entry => SerializedAssetGraphNode::Entry, + AssetGraphNode::Asset(asset_node) => SerializedAssetGraphNode::Asset { + value: asset_node.asset.clone(), + }, + AssetGraphNode::Dependency(dependency_node) => SerializedAssetGraphNode::Dependency { + value: SerializedDependency { + id: dependency_node.dependency.id(), + dependency: dependency_node.dependency.as_ref().clone(), + }, + has_deferred: dependency_node.state == DependencyState::Deferred, + }, + }; + + let node_bytes = serde_json::to_vec(&serialized_node)?; + let js_buffer = env.create_buffer_with_data(node_bytes)?; + let js_buffer = js_buffer.into_unknown(); + + js_nodes.set_element(i as u32, js_buffer)?; + } + + let mut napi_asset_graph = env.create_object()?; + napi_asset_graph.set_named_property("edges", asset_graph.edges())?; + napi_asset_graph.set_named_property("nodes", js_nodes)?; + + Ok(napi_asset_graph) +} diff --git a/packages/core/core/src/requests/AssetGraphRequestRust.js b/packages/core/core/src/requests/AssetGraphRequestRust.js index f6dc42424..d0452ffb0 100644 --- a/packages/core/core/src/requests/AssetGraphRequestRust.js +++ b/packages/core/core/src/requests/AssetGraphRequestRust.js @@ -39,7 +39,7 @@ export function createAssetGraphRequestRust( let options = input.options; let serializedAssetGraph = await rustAtlaspack.buildAssetGraph(); - serializedAssetGraph.nodes = serializedAssetGraph.nodes.flatMap((node) => + serializedAssetGraph.nodes = serializedAssetGraph.nodes.map((node) => JSON.parse(node), ); From d69cc34efd153731172be0cc9064624901f15318 Mon Sep 17 00:00:00 2001 From: David Alsh Date: Wed, 22 Jan 2025 08:18:06 +0000 Subject: [PATCH 2/2] Serialize in parallel --- .../src/atlaspack/serialize_asset_graph.rs | 89 +++++++++++-------- 1 file changed, 50 insertions(+), 39 deletions(-) diff --git a/crates/node-bindings/src/atlaspack/serialize_asset_graph.rs b/crates/node-bindings/src/atlaspack/serialize_asset_graph.rs index ba08c470b..3afe94c5c 100644 --- a/crates/node-bindings/src/atlaspack/serialize_asset_graph.rs +++ b/crates/node-bindings/src/atlaspack/serialize_asset_graph.rs @@ -1,30 +1,10 @@ use napi::{Env, JsObject}; +use rayon::iter::{IntoParallelRefIterator, ParallelIterator}; use serde::Serialize; use atlaspack_core::asset_graph::{AssetGraph, AssetGraphNode, DependencyState}; use atlaspack_core::types::{Asset, Dependency}; -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct SerializedDependency { - id: String, - dependency: Dependency, -} - -#[derive(Debug, Serialize)] -#[serde(tag = "type", rename_all = "camelCase")] -enum SerializedAssetGraphNode { - Root, - Entry, - Asset { - value: Asset, - }, - Dependency { - value: SerializedDependency, - has_deferred: bool, - }, -} - /// Returns /// ```typescript /// type SerializedAssetGraph = { @@ -33,34 +13,65 @@ enum SerializedAssetGraphNode { /// } /// ``` pub fn serialize_asset_graph(env: &Env, asset_graph: &AssetGraph) -> anyhow::Result { - let mut js_nodes = env.create_empty_array()?; - - for (i, node) in asset_graph.nodes().enumerate() { - let serialized_node = match node { - AssetGraphNode::Root => SerializedAssetGraphNode::Root, - AssetGraphNode::Entry => SerializedAssetGraphNode::Entry, - AssetGraphNode::Asset(asset_node) => SerializedAssetGraphNode::Asset { - value: asset_node.asset.clone(), - }, - AssetGraphNode::Dependency(dependency_node) => SerializedAssetGraphNode::Dependency { - value: SerializedDependency { - id: dependency_node.dependency.id(), - dependency: dependency_node.dependency.as_ref().clone(), + // Serialize graph nodes in parallel + let nodes = asset_graph + .nodes() + .collect::>() + .par_iter() + .map(|item| { + serde_json::to_vec(&match item { + AssetGraphNode::Root => SerializedAssetGraphNode::Root, + AssetGraphNode::Entry => SerializedAssetGraphNode::Entry, + AssetGraphNode::Asset(asset_node) => SerializedAssetGraphNode::Asset { + value: asset_node.asset.clone(), + }, + AssetGraphNode::Dependency(dependency_node) => SerializedAssetGraphNode::Dependency { + value: SerializedDependency { + id: dependency_node.dependency.id(), + dependency: dependency_node.dependency.as_ref().clone(), + }, + has_deferred: dependency_node.state == DependencyState::Deferred, }, - has_deferred: dependency_node.state == DependencyState::Deferred, - }, - }; + }) + }) + .collect::>(); - let node_bytes = serde_json::to_vec(&serialized_node)?; - let js_buffer = env.create_buffer_with_data(node_bytes)?; + // Collect the results into a JavaScript Array + // TODO NAPI3 allows removing this + let mut js_nodes = env.create_array_with_length(nodes.len())?; + + for (i, node_bytes) in nodes.into_iter().enumerate() { + let js_buffer = env.create_buffer_with_data(node_bytes?)?; let js_buffer = js_buffer.into_unknown(); js_nodes.set_element(i as u32, js_buffer)?; } + // TODO use a napi_derive::napi(object) with NAPI3 let mut napi_asset_graph = env.create_object()?; napi_asset_graph.set_named_property("edges", asset_graph.edges())?; napi_asset_graph.set_named_property("nodes", js_nodes)?; Ok(napi_asset_graph) } + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct SerializedDependency { + id: String, + dependency: Dependency, +} + +#[derive(Debug, Serialize)] +#[serde(tag = "type", rename_all = "camelCase")] +enum SerializedAssetGraphNode { + Root, + Entry, + Asset { + value: Asset, + }, + Dependency { + value: SerializedDependency, + has_deferred: bool, + }, +}