From 75f9ad0eca894517f626c4a6eb5be3faf19eb9e3 Mon Sep 17 00:00:00 2001 From: elrrrrrrr Date: Tue, 21 Oct 2025 18:57:58 +0800 Subject: [PATCH 1/3] refactor: pet --- crates/pm/src/helper/graph_builder.rs | 399 ++++++++ crates/pm/src/helper/lock.rs | 809 +-------------- crates/pm/src/helper/mod.rs | 1 + crates/pm/src/helper/ruborist.rs | 918 +++--------------- crates/pm/src/model/graph.rs | 880 +++++++++++++++++ crates/pm/src/model/mod.rs | 1 + crates/pm/src/model/node.rs | 324 +------ crates/pm/src/model/override_rule.rs | 62 +- .../pm/src/service/dependency_resolution.rs | 68 +- crates/pm/src/service/mod.rs | 1 + crates/pm/src/service/preload.rs | 520 ++++++++++ crates/pm/src/service/workspace.rs | 35 +- crates/pm/src/util/cloner.rs | 2 +- crates/pm/src/util/mod.rs | 1 - crates/pm/src/util/node_search.rs | 106 -- crates/pm/src/util/semver.rs | 4 - e2e/utoo-pm.sh | 4 +- 17 files changed, 1995 insertions(+), 2140 deletions(-) create mode 100644 crates/pm/src/helper/graph_builder.rs create mode 100644 crates/pm/src/model/graph.rs create mode 100644 crates/pm/src/service/preload.rs delete mode 100644 crates/pm/src/util/node_search.rs diff --git a/crates/pm/src/helper/graph_builder.rs b/crates/pm/src/helper/graph_builder.rs new file mode 100644 index 000000000..9439090bb --- /dev/null +++ b/crates/pm/src/helper/graph_builder.rs @@ -0,0 +1,399 @@ +use anyhow::Result; +use petgraph::graph::{EdgeIndex, NodeIndex}; + +use crate::model::graph::{DependencyGraph, FindResult, PackageNode}; +use crate::model::node::EdgeType; +use crate::util::config::get_legacy_peer_deps; +use crate::util::logger::{PROGRESS_BAR, log_progress}; +use crate::util::registry::{ResolvedPackage, resolve_dependency}; + +/// Represents dependency edge information extracted from the graph +/// This structure improves readability compared to using tuples +#[derive(Debug, Clone)] +struct DependencyEdgeInfo { + edge_id: EdgeIndex, + name: String, + spec: String, + edge_type: EdgeType, + is_valid: bool, + target: Option, +} + +/// Represents node type information for dependency propagation +/// Used to avoid borrowing conflicts when updating node types +#[derive(Debug, Clone, Copy)] +struct NodeTypeInfo { + is_root: bool, + is_prod: bool, + is_dev: bool, + is_optional: bool, +} + +/// Collect dependency edges from a node and convert to structured info +/// This helper function improves code readability by avoiding complex tuple destructuring +fn collect_dependency_edges( + graph: &DependencyGraph, + node_index: NodeIndex, +) -> Vec { + let dep_edges = graph.get_dependency_edges(node_index); + dep_edges + .into_iter() + .map(|(edge_id, dep)| DependencyEdgeInfo { + edge_id, + name: dep.name.clone(), + spec: dep.spec.clone(), + edge_type: dep.edge_type.clone(), + is_valid: dep.valid, + target: dep.to, + }) + .collect() +} + +/// Build dependency tree using single-threaded BFS traversal +pub async fn build_deps(graph: &mut DependencyGraph) -> Result<()> { + let legacy_peer_deps = get_legacy_peer_deps().await; + tracing::debug!("going to build deps for root, legacy_peer_deps: {legacy_peer_deps}"); + + let mut current_level = vec![graph.root_index]; + + while !current_level.is_empty() { + let level_count = current_level.len(); + tracing::debug!("Starting new dependency level with {level_count} nodes"); + + let mut next_level = Vec::new(); + + for node_index in current_level { + // Collect all dependency edges from this node (both valid and invalid) + // We extract edge info into a dedicated struct to avoid borrow conflicts + // and improve code readability + let dependency_edges = collect_dependency_edges(graph, node_index); + + // Count unresolved edges for progress bar tracking + let unresolved_count = dependency_edges + .iter() + .filter(|edge| !edge.is_valid) + .count(); + PROGRESS_BAR.inc_length(unresolved_count as u64); + log_progress(&format!( + "resolving {}", + graph.get_node(node_index).unwrap().name + )); + + for edge_info in dependency_edges { + // Handle already resolved edges (e.g., workspace edges) + if edge_info.is_valid { + if let Some(target_idx) = edge_info.target { + let target_node = graph.get_node(target_idx).unwrap(); + // Only add workspace nodes to next level when from root + if target_node.is_workspace() && node_index == graph.root_index { + next_level.push(target_idx); + } + } + continue; + } + + tracing::debug!( + "going to build deps {}@{} from [{:?}]", + edge_info.name, + edge_info.spec, + node_index + ); + + let start_time = std::time::Instant::now(); + + // Find installation location + match graph.find_compatible_node(node_index, &edge_info.name, &edge_info.spec) { + FindResult::Reuse(existing_index) => { + tracing::debug!( + "resolved deps {}@{} => {} (reuse) took {:?}", + edge_info.name, + edge_info.spec, + graph.get_node(existing_index).unwrap().version, + start_time.elapsed() + ); + + // Mark edge as resolved + graph.mark_dependency_resolved(edge_info.edge_id, existing_index); + + // Update target node type based on edge type + update_node_type_from_edge( + graph, + node_index, + existing_index, + &edge_info.edge_type, + ); + } + FindResult::Conflict(conflict_parent) | FindResult::New(conflict_parent) => { + tracing::debug!( + "Conflict/New found for {}@{}, resolving...", + edge_info.name, + edge_info.spec + ); + + // Check if there's an override for this dependency + let effective_spec = graph + .check_override(node_index, &edge_info.name, &edge_info.spec) + .unwrap_or_else(|| edge_info.spec.clone()); + + // Resolve dependency from registry with effective spec + let resolved = match resolve_dependency( + &edge_info.name, + &effective_spec, + &edge_info.edge_type, + ) + .await? + { + Some(resolved) => { + tracing::debug!( + "Resolved dependency {}@{} => {}", + edge_info.name, + edge_info.spec, + resolved.version + ); + resolved + } + None => { + tracing::debug!( + "No resolution found for {}@{}", + edge_info.name, + edge_info.spec + ); + PROGRESS_BAR.inc(1); + continue; + } + }; + + PROGRESS_BAR.inc(1); + tracing::debug!( + "resolved deps {}@{} => {} (conflict/new) took {:?}", + edge_info.name, + edge_info.spec, + resolved.version, + start_time.elapsed() + ); + + // Create new node + let new_node = place_deps( + edge_info.name.clone(), + resolved.clone(), + conflict_parent, + graph, + ); + let new_index = graph.add_node(new_node); + + // Add physical edge + graph.add_physical_edge(conflict_parent, new_index); + + // Mark edge as resolved + graph.mark_dependency_resolved(edge_info.edge_id, new_index); + + // Update target node type based on edge type + update_node_type_from_edge( + graph, + node_index, + new_index, + &edge_info.edge_type, + ); + + // Add dependencies of the new node + add_dependency_edges( + graph, + new_index, + &resolved.manifest, + legacy_peer_deps, + ) + .await; + + // Add to next level for processing + next_level.push(new_index); + } + } + } + } + + tracing::debug!("Level completed, next level has {} nodes", next_level.len()); + current_level = next_level; + } + + Ok(()) +} + +/// Create a new package node +fn place_deps( + name: String, + pkg: ResolvedPackage, + parent: NodeIndex, + graph: &DependencyGraph, +) -> PackageNode { + let parent_node = graph.get_node(parent).unwrap(); + let path = if parent_node.path.to_string_lossy().is_empty() + || parent_node.path == std::path::Path::new(".") + { + std::path::PathBuf::from(format!("node_modules/{name}")) + } else { + parent_node.path.join(format!("node_modules/{name}")) + }; + + let new_node = PackageNode::new(name.clone(), path, pkg.manifest); + + tracing::debug!( + "\nInstalling {}@{} under parent {:?}", + new_node.name, + new_node.version, + parent + ); + + new_node +} + +/// Add dependency edges for a node +async fn add_dependency_edges( + graph: &mut DependencyGraph, + node_index: NodeIndex, + manifest: &serde_json::Value, + legacy_peer_deps: bool, +) { + let dep_types = if legacy_peer_deps { + vec![ + ("dependencies", EdgeType::Prod), + ("optionalDependencies", EdgeType::Optional), + ] + } else { + vec![ + ("dependencies", EdgeType::Prod), + ("peerDependencies", EdgeType::Peer), + ("optionalDependencies", EdgeType::Optional), + ] + }; + + for (field, edge_type) in dep_types { + if let Some(deps) = manifest.get(field).and_then(|v| v.as_object()) { + tracing::debug!( + "Processing {} dependencies for {}", + field, + graph.get_node(node_index).unwrap().name + ); + for (name, version) in deps { + let version_spec = version.as_str().unwrap_or("").to_string(); + graph.add_dependency_edge( + node_index, + name.clone(), + version_spec, + edge_type.clone(), + ); + tracing::debug!( + "add edge {}@{} for {}", + name, + version, + graph.get_node(node_index).unwrap().name + ); + } + tracing::debug!( + "Finished processing {} dependencies for {}", + field, + graph.get_node(node_index).unwrap().name + ); + } + } +} + +/// Update target node type based on source node and edge type +/// +/// This function propagates dependency types through the graph according to npm rules: +/// - Root dependencies directly set the target node type +/// - Prod dependencies propagate through prod edges +/// - Dev/Optional flags propagate only when appropriate +fn update_node_type_from_edge( + graph: &mut DependencyGraph, + from_index: NodeIndex, + to_index: NodeIndex, + edge_type: &EdgeType, +) { + // Extract source node information to avoid borrowing conflicts + // We need these flags to determine how to mark the target node + let source_node_info = { + let from_node = graph.get_node(from_index).unwrap(); + NodeTypeInfo { + is_root: from_node.is_root(), + is_prod: from_node.is_prod, + is_dev: from_node.is_dev, + is_optional: from_node.is_optional, + } + }; + + // Update target node based on source node type and edge type + let to_node = graph.get_node_mut(to_index).unwrap(); + + // Root node dependencies directly determine target type + if source_node_info.is_root { + match edge_type { + EdgeType::Prod => { + to_node.is_prod = true; + to_node.is_dev = false; + to_node.is_optional = false; + to_node.is_peer = false; + } + EdgeType::Dev => { + if !to_node.is_prod { + to_node.is_dev = true; + to_node.is_optional = false; + } + } + EdgeType::Optional => { + if !to_node.is_prod && !to_node.is_dev { + to_node.is_optional = true; + } + } + EdgeType::Peer => { + if !to_node.is_prod && !to_node.is_dev { + to_node.is_peer = true; + } + } + } + } else { + // If source is prod, and edge is prod, target must be prod + if source_node_info.is_prod && *edge_type == EdgeType::Prod { + to_node.is_prod = true; + to_node.is_dev = false; + to_node.is_optional = false; + to_node.is_peer = false; + } + // Propagate dev + else if source_node_info.is_dev && *edge_type == EdgeType::Dev && !to_node.is_prod { + to_node.is_dev = true; + } + // Propagate optional + else if source_node_info.is_optional + && *edge_type == EdgeType::Optional + && !to_node.is_prod + && !to_node.is_dev + { + to_node.is_optional = true; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + use std::path::PathBuf; + + #[test] + fn test_place_deps() { + let pkg = json!({"name": "root", "version": "1.0.0"}); + let graph = DependencyGraph::new(PathBuf::from("."), pkg); + + let resolved = ResolvedPackage { + name: "lodash".to_string(), + version: "4.17.21".to_string(), + manifest: json!({"name": "lodash", "version": "4.17.21"}), + }; + + let new_node = place_deps("lodash".to_string(), resolved, graph.root_index, &graph); + + assert_eq!(new_node.name, "lodash"); + assert_eq!(new_node.version, "4.17.21"); + assert_eq!(new_node.path, PathBuf::from("node_modules/lodash")); + } +} diff --git a/crates/pm/src/helper/lock.rs b/crates/pm/src/helper/lock.rs index a04348849..bc0855069 100644 --- a/crates/pm/src/helper/lock.rs +++ b/crates/pm/src/helper/lock.rs @@ -3,19 +3,16 @@ use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use std::collections::HashMap; use std::path::{Path, PathBuf}; -use std::sync::Arc; use crate::helper::workspace::find_workspaces; -use crate::model::node::{EdgeType, Node}; -use crate::model::override_rule::Overrides; +use crate::model::graph::DependencyGraph; use crate::model::package_lock::LockPackage; use crate::service::dependency_resolution::DependencyResolutionService; use crate::util::config::get_legacy_peer_deps; use crate::util::json::{load_package_json_from_path, load_package_lock_json_from_path}; -use crate::util::registry::{resolve, resolve_dependency}; +use crate::util::registry::resolve; use crate::util::relative_path::to_relative_path; use crate::util::save_type::{PackageAction, SaveType}; -use crate::util::semver; use crate::util::{cache::parse_pattern, cloner::clone, downloader::download}; use super::workspace::find_workspace_path; @@ -306,12 +303,6 @@ pub fn path_to_pkg_name(path_str: &str) -> Option<&str> { } } -#[derive(Debug)] -pub struct InvalidDependency { - pub package_path: String, - pub dependency_name: String, -} - pub async fn is_pkg_lock_outdated(root_path: &Path) -> Result { let pkg_file = load_package_json_from_path(root_path).await?; let lock_file = load_package_lock_json_from_path(root_path).await?; @@ -362,137 +353,6 @@ pub async fn is_pkg_lock_outdated(root_path: &Path) -> Result { Ok(false) } -pub async fn validate_deps( - pkg_file: &Value, - pkgs_in_pkg_lock: &Value, -) -> Result> { - let mut invalid_deps = Vec::new(); - // Initialize overrides - let overrides = Overrides::parse(pkg_file.clone()); - - if let Some(packages) = pkgs_in_pkg_lock.as_object() { - for (pkg_path, pkg_info) in packages { - for (dep_field, is_optional) in get_dep_types().await { - if let Some(dependencies) = pkg_info.get(dep_field).and_then(|d| d.as_object()) { - for (dep_name, req_version) in dependencies { - let req_version_str = req_version.as_str().unwrap_or_default(); - - // Collect parent chain information - let mut parent_chain = Vec::new(); - let mut current_path = String::from(pkg_path); - - while !current_path.is_empty() { - if let Some(pkg_info) = packages.get(¤t_path) - && let Some(name) = pkg_info.get("name").and_then(|n| n.as_str()) - && let Some(version) = - pkg_info.get("version").and_then(|v| v.as_str()) - { - parent_chain.push((name.to_string(), version.to_string())); - } - - if let Some(last_modules) = current_path.rfind("/node_modules/") { - current_path = current_path[..last_modules].to_string(); - } else { - current_path = String::new(); - } - } - - // Check if there's an override rule for this dependency - let effective_req_version = if let Some(overrides) = &overrides { - let mut effective_version = req_version_str.to_string(); - for rule in &overrides.rules { - // Clone the rule to avoid holding the lock across await - let rule = rule.clone(); - let matches = overrides - .matches_rule(&rule, dep_name, req_version_str, &parent_chain) - .await; - if matches { - effective_version = rule.target_spec.clone(); - break; - } - } - effective_version - } else { - req_version_str.to_string() - }; - - // find the actual version of the dependency - let mut current_path = String::from(pkg_path); - let mut dep_info = None; - - // until root or found - loop { - let search_path = if current_path.is_empty() { - format!("node_modules/{dep_name}") - } else { - format!("{current_path}/node_modules/{dep_name}") - }; - - if let Some(info) = packages.get(&search_path) { - dep_info = Some(info); - current_path = search_path; - break; - } - - // find in root path - if current_path.is_empty() { - break; - } - - // find in parent path - if let Some(last_modules) = current_path.rfind("/node_modules/") { - current_path = current_path[..last_modules].to_string(); - } else { - current_path = String::new(); - } - } - - // optional dependency not found is allowed - if let Some(dep_info) = dep_info { - if let Some(actual_version) = - dep_info.get("version").and_then(|v| v.as_str()) - && !semver::matches(&effective_req_version, actual_version) - { - if let Some(resolved_dep) = resolve_dependency( - dep_name, - &effective_req_version, - &EdgeType::Optional, - ) - .await? - && resolved_dep.version == actual_version - { - tracing::debug!( - "Package {pkg_path} {dep_field} dependency {dep_name} (required version: {req_version_str}, effective version: {effective_req_version}) hit bug-version {current_path}@{actual_version}" - ); - continue; - } - - tracing::debug!( - "Package {pkg_path} {dep_field} dependency {dep_name} (required version: {req_version_str}, effective version: {effective_req_version}) does not match actual version {current_path}@{actual_version}" - ); - invalid_deps.push(InvalidDependency { - package_path: pkg_path.to_string(), - dependency_name: dep_name.to_string(), - }); - } - } else if !is_optional { - tracing::debug!( - "pkg_path {pkg_path} dep_field {dep_field} dep_name {dep_name} not found" - ); - invalid_deps.push(InvalidDependency { - package_path: pkg_path.to_string(), - dependency_name: dep_name.to_string(), - }); - } - } - } - } - } - } - - Ok(invalid_deps) -} - async fn get_dep_types() -> Vec<(&'static str, bool)> { let legacy_peer_deps = get_legacy_peer_deps().await; @@ -520,9 +380,9 @@ fn convert_packages_to_hashmap(packages: Value) -> Result, + graph: &DependencyGraph, ) -> Result { - let (packages, total_packages) = serialize_tree_to_packages(ideal_tree, path); + let (packages, total_packages) = graph.serialize_to_packages(path); tracing::debug!("Total {total_packages} dependencies after merging"); @@ -569,246 +429,9 @@ pub async fn save_package_lock(path: &Path, package_lock: &PackageLock) -> Resul Ok(()) } -pub fn serialize_tree_to_packages(node: &Arc, path: &Path) -> (Value, i32) { - let mut packages = json!({}); - let mut stack = vec![(node.clone(), String::new())]; - let mut total_packages = 0; - - while let Some((current, prefix)) = stack.pop() { - // Check for duplicate dependencies - check_duplicate_dependencies(¤t); - - // Create package info based on node type - let pkg_info = create_package_info(¤t, path, &mut total_packages); - - // Use empty string for root node - let key = if prefix.is_empty() { - String::new() - } else { - prefix.clone() - }; - packages[key] = pkg_info; - - // Add children to processing stack - add_children_to_stack(¤t, &prefix, path, &mut stack); - } - - (packages, total_packages) -} - -/// Check for duplicate dependencies under a node and log warnings -fn check_duplicate_dependencies(node: &Arc) { - let children = node.children.read().unwrap(); - let mut name_count = HashMap::new(); - - for child in children.iter() { - if !child.is_link() { - *name_count.entry(child.name.as_str()).or_insert(0) += 1; - } - } - - for (name, count) in name_count { - if count > 1 { - tracing::warn!( - "Found {} duplicate dependencies named '{}' under '{}'", - count, - name, - node.name - ); - } - } -} - -/// Create package information based on node type -fn create_package_info(node: &Arc, root_path: &Path, total_packages: &mut i32) -> Value { - let mut pkg_info = if node.is_root() { - create_root_package_info(node) - } else { - create_non_root_package_info(node, root_path, total_packages) - }; - - // Add package fields (dependencies, bin, license, etc.) - add_package_fields(&mut pkg_info, node); - - pkg_info -} - -/// Create package info for root node -fn create_root_package_info(node: &Arc) -> Value { - let mut info = json!({ - "name": node.name, - "version": node.version, - }); - - if let Some(engines) = node.package.get("engines") { - info["engines"] = engines.clone(); - } - - if let Some(workspaces) = node.package.get("workspaces") { - info["workspaces"] = workspaces.clone(); - } - - info -} - -/// Create package info for non-root nodes -fn create_non_root_package_info( - node: &Arc, - root_path: &Path, - total_packages: &mut i32, -) -> Value { - let mut info = json!({ - "name": node.package.get("name"), - }); - - if node.is_workspace() { - info["version"] = json!(node.package.get("version")); - } else if node.is_link() { - info["link"] = json!(true); - let target_path = get_relative_target_path(node, root_path); - info["resolved"] = json!(target_path); - } else { - // Regular package - info["version"] = json!(node.package.get("version")); - - let empty_dist = json!(""); - let dist = node.package.get("dist").unwrap_or(&empty_dist); - info["resolved"] = json!(dist.get("tarball")); - info["integrity"] = json!(dist.get("integrity")); - - *total_packages += 1; - } - - // Add optional flags - add_optional_flags(&mut info, node); - - info -} - -/// Add optional flags (peer, dev, optional, hasInstallScript) -fn add_optional_flags(info: &mut Value, node: &Arc) { - if *node.is_peer.read().unwrap() == Some(true) { - info["peer"] = json!(true); - } - - let is_dev = *node.is_dev.read().unwrap() == Some(true); - let is_optional = *node.is_optional.read().unwrap() == Some(true); - - match (is_dev, is_optional) { - (true, true) => info["devOptional"] = json!(true), - (true, false) => info["dev"] = json!(true), - (false, true) => info["optional"] = json!(true), - _ => {} - } - - if node.package.get("hasInstallScript") == Some(&json!(true)) { - info["hasInstallScript"] = json!(true); - } -} - -/// Add package fields based on node type -fn add_package_fields(pkg_info: &mut Value, node: &Arc) { - let fields = get_package_fields(node); - - for field in fields { - if let Some(field_value) = node.package.get(field) - && should_include_field(field_value) - { - pkg_info[field] = field_value.clone(); - } - } -} - -/// Get the list of fields to include based on node type -fn get_package_fields(node: &Arc) -> Vec<&'static str> { - if node.is_link() { - vec![] - } else if node.is_root() { - vec![ - "dependencies", - "devDependencies", - "peerDependencies", - "optionalDependencies", - ] - } else { - let mut fields = vec![ - "dependencies", - "peerDependencies", - "optionalDependencies", - "bin", - "license", - "engines", - "os", - "cpu", - ]; - - if node.is_workspace() { - fields.push("devDependencies"); - } - - fields - } -} - -/// Check if a field value should be included in the output -fn should_include_field(field_value: &Value) -> bool { - if field_value.is_object() { - !field_value.as_object().unwrap().is_empty() - } else { - true // Include non-object values (strings, etc.) - } -} - -/// Add children to the processing stack -fn add_children_to_stack( - node: &Arc, - prefix: &str, - root_path: &Path, - stack: &mut Vec<(Arc, String)>, -) { - let children = node.children.read().unwrap(); - - for child in children.iter() { - let child_prefix = generate_child_prefix(prefix, child, root_path); - stack.push((child.clone(), child_prefix)); - } -} - -/// Generate the prefix path for a child node -fn generate_child_prefix(prefix: &str, child: &Arc, root_path: &Path) -> String { - if prefix.is_empty() { - if child.is_workspace() { - // Convert workspace path to relative path - child - .path - .strip_prefix(root_path) - .map(|p| p.to_string_lossy().to_string()) - .unwrap_or_else(|_| child.path.to_string_lossy().to_string()) - } else { - format!("node_modules/{}", child.name) - } - } else { - format!("{}/node_modules/{}", prefix, child.name) - } -} - -/// Get the relative path of a link target from the root path -fn get_relative_target_path(current: &Node, root_path: &Path) -> String { - let target = current.target.read().unwrap(); - let target_node = target.as_ref().unwrap(); - - // Try to get relative path first - target_node - .path - .strip_prefix(root_path) - .map(|p| p.to_string_lossy().to_string()) - .unwrap_or_else(|_| target_node.path.to_string_lossy().to_string()) -} - #[cfg(test)] mod tests { use super::*; - use crate::model::node::Node; use serde_json::json; use std::fs; use tempfile::TempDir; @@ -860,114 +483,6 @@ mod tests { ); } - #[tokio::test] - async fn test_validate_deps_with_invalid_dependencies() { - // Create a mock package.json - let pkg_file = json!({ - "name": "test-package", - "version": "1.0.0", - "dependencies": { - "lodash": "^4.17.20" - } - }); - - // Create a mock package-lock.json structure - let pkgs_in_pkg_lock = json!({ - "": { - "name": "test-package", - "version": "1.0.0", - "dependencies": { - "lodash": "^4.17.20" - } - }, - "node_modules/lodash": { - "name": "lodash", - "version": "3.17.20", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-3.17.20.tgz" - } - }); - - let invalid_deps = validate_deps(&pkg_file, &pkgs_in_pkg_lock).await.unwrap(); - assert!(!invalid_deps.is_empty()); - } - - #[tokio::test] - async fn test_validate_deps_with_valid_dependencies() { - // Create a mock package.json - let pkg_file = json!({ - "name": "test-package", - "version": "1.0.0", - "dependencies": { - "lodash": "^4.17.20" - } - }); - - // Create a mock package-lock.json structure - let pkgs_in_pkg_lock = json!({ - "": { - "name": "test-package", - "version": "1.0.0", - "dependencies": { - "lodash": "^4.17.20" - } - }, - "node_modules/lodash": { - "name": "lodash", - "version": "4.17.20", - "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz" - } - }); - - let invalid_deps = validate_deps(&pkg_file, &pkgs_in_pkg_lock).await.unwrap(); - assert!(invalid_deps.is_empty()); - } - - #[tokio::test] - async fn test_build_ideal_tree_to_package_lock() { - // Create a temporary directory for testing - let temp_dir = TempDir::new().unwrap(); - let temp_path = temp_dir.path(); - - // Create a mock ideal tree - let root = Node::new( - "test-package".to_string(), - temp_path.to_path_buf(), - json!({ - "name": "test-package", - "version": "1.0.0", - "dependencies": { - "lodash": "^4.17.20" - } - }), - ); - - // Test building PackageLock from ideal tree - let result = build_ideal_tree_to_package_lock(temp_path, &root).await; - assert!(result.is_ok()); - - let package_lock = result.unwrap(); - - // Verify PackageLock structure - assert!(package_lock.packages.contains_key("")); - - // Save to disk synchronously - save_package_lock(temp_path, &package_lock).await.unwrap(); - - // Verify the lock file was created on disk - let lock_file_path = temp_path.join("package-lock.json"); - assert!(lock_file_path.exists()); - - // Read and verify the content - let content = fs::read_to_string(lock_file_path).unwrap(); - let lock_data: Value = serde_json::from_str(&content).unwrap(); - - assert_eq!(lock_data["name"], "test-package"); - - // Verify packages section - let packages = lock_data["packages"].as_object().unwrap(); - assert!(packages.contains_key("")); - } - #[tokio::test] async fn test_is_pkg_lock_outdated() { // Create a temporary directory @@ -1085,322 +600,6 @@ mod tests { assert!(is_pkg_lock_outdated(temp_path).await.unwrap()); } - #[test] - fn test_get_relative_target_path() { - // Create a temporary directory for testing - let temp_dir = TempDir::new().unwrap(); - let root_path = temp_dir.path(); - - // Test case 1: Target path is under root path - let target_path = root_path.join("packages/pkg-a"); - let node = Node::new( - "test-package".to_string(), - root_path.to_path_buf(), - json!({ - "name": "test-package", - "version": "1.0.0" - }), - ); - node.target.write().unwrap().replace(Node::new( - "target-package".to_string(), - target_path.clone(), - json!({ - "name": "target-package", - "version": "1.0.0" - }), - )); - - let relative_path = get_relative_target_path(&node, root_path); - assert_eq!(relative_path, "packages/pkg-a"); - - // Test case 2: Target path is outside root path - let outside_path = PathBuf::from("/some/outside/path"); - let node = Node::new( - "test-package".to_string(), - root_path.to_path_buf(), - json!({ - "name": "test-package", - "version": "1.0.0" - }), - ); - node.target.write().unwrap().replace(Node::new( - "target-package".to_string(), - outside_path.clone(), - json!({ - "name": "target-package", - "version": "1.0.0" - }), - )); - - let relative_path = get_relative_target_path(&node, root_path); - assert_eq!(relative_path, "/some/outside/path"); - - // Test case 3: Target path is the root path - let node = Node::new( - "test-package".to_string(), - root_path.to_path_buf(), - json!({ - "name": "test-package", - "version": "1.0.0" - }), - ); - node.target.write().unwrap().replace(Node::new( - "target-package".to_string(), - root_path.to_path_buf(), - json!({ - "name": "target-package", - "version": "1.0.0" - }), - )); - - let relative_path = get_relative_target_path(&node, root_path); - assert_eq!(relative_path, ""); - } - - #[test] - fn test_serialize_tree_to_packages_with_workspace_bin() { - // Create a temporary directory for testing - let temp_dir = TempDir::new().unwrap(); - let temp_path = temp_dir.path(); - - // Create root package.json - fs::write( - temp_path.join("package.json"), - json!({ - "name": "test-package", - "version": "1.0.0", - "workspaces": ["packages/*"] - }) - .to_string(), - ) - .unwrap(); - - // Create workspace package directory and package.json - let workspace_path = temp_path.join("packages/workspace-a"); - fs::create_dir_all(&workspace_path).unwrap(); - fs::write( - workspace_path.join("package.json"), - json!({ - "name": "workspace-a", - "version": "1.0.0", - "bin": { - "workspace-a": "./bin/index.js", - "workspace-a-cli": "./bin/cli.js" - } - }) - .to_string(), - ) - .unwrap(); - - // Create bin directory and files - let bin_path = workspace_path.join("bin"); - fs::create_dir_all(&bin_path).unwrap(); - fs::write(bin_path.join("index.js"), "console.log('workspace-a');").unwrap(); - fs::write(bin_path.join("cli.js"), "console.log('workspace-a-cli');").unwrap(); - - // Create root node - let root = Node::new( - "test-package".to_string(), - temp_path.to_path_buf(), - json!({ - "name": "test-package", - "version": "1.0.0", - "workspaces": ["packages/*"] - }), - ); - - // Create workspace node - let workspace = Node::new_workspace( - "workspace-a".to_string(), - workspace_path.clone(), - json!({ - "name": "workspace-a", - "version": "1.0.0", - "bin": { - "workspace-a": "./bin/index.js", - "workspace-a-cli": "./bin/cli.js" - } - }), - ); - root.children.write().unwrap().push(workspace); - - // Test serialization - let (packages, _) = serialize_tree_to_packages(&root, temp_path); - - // Verify workspace package - let workspace_pkg = packages.get("packages/workspace-a").unwrap(); - println!("workspace_pkg: {workspace_pkg:?}"); - assert_eq!(workspace_pkg["name"], "workspace-a"); - - // Verify bin configuration - let bin = workspace_pkg["bin"].as_object().unwrap(); - assert_eq!(bin["workspace-a"], "./bin/index.js"); - assert_eq!(bin["workspace-a-cli"], "./bin/cli.js"); - } - - #[tokio::test] - async fn test_validate_deps_with_version_mismatch() { - // Create a mock package.json - let pkg_file = json!({ - "name": "test-package", - "version": "1.0.0", - "dependencies": { - "example-pkg": "^2.0.0" - } - }); - - // Create a mock package-lock.json structure with version mismatch - let pkgs_in_pkg_lock = json!({ - "": { - "name": "test-package", - "version": "1.0.0", - "dependencies": { - "example-pkg": "^2.0.0" - } - }, - "node_modules/example-pkg": { - "name": "example-pkg", - "version": "1.5.0", // Doesn't match ^2.0.0 - "resolved": "https://registry.npmjs.org/example-pkg/-/example-pkg-1.5.0.tgz" - } - }); - - let invalid_deps = validate_deps(&pkg_file, &pkgs_in_pkg_lock).await.unwrap(); - assert_eq!(invalid_deps.len(), 1); - assert_eq!(invalid_deps[0].package_path, ""); - assert_eq!(invalid_deps[0].dependency_name, "example-pkg"); - } - - #[tokio::test] - async fn test_validate_deps_with_optional_dependency_missing() { - // Create a mock package.json - let pkg_file = json!({ - "name": "test-package", - "version": "1.0.0", - "optionalDependencies": { - "optional-pkg": "^1.0.0" - } - }); - - // Create a mock package-lock.json structure with missing optional dependency - let pkgs_in_pkg_lock = json!({ - "": { - "name": "test-package", - "version": "1.0.0", - "optionalDependencies": { - "optional-pkg": "^1.0.0" - } - } - // optional-pkg is missing from node_modules - }); - - let invalid_deps = validate_deps(&pkg_file, &pkgs_in_pkg_lock).await.unwrap(); - // Optional dependency missing should not be treated as invalid - assert_eq!(invalid_deps.len(), 0); - } - - #[tokio::test] - async fn test_validate_deps_with_required_dependency_missing() { - // Create a mock package.json - let pkg_file = json!({ - "name": "test-package", - "version": "1.0.0", - "dependencies": { - "required-pkg": "^1.0.0" - } - }); - - // Create a mock package-lock.json structure with missing required dependency - let pkgs_in_pkg_lock = json!({ - "": { - "name": "test-package", - "version": "1.0.0", - "dependencies": { - "required-pkg": "^1.0.0" - } - } - // required-pkg is missing from node_modules - }); - - let invalid_deps = validate_deps(&pkg_file, &pkgs_in_pkg_lock).await.unwrap(); - // Required dependency missing should be treated as invalid - assert_eq!(invalid_deps.len(), 1); - assert_eq!(invalid_deps[0].package_path, ""); - assert_eq!(invalid_deps[0].dependency_name, "required-pkg"); - } - - #[tokio::test] - async fn test_validate_deps_with_peer_dependencies() { - // Create a mock package.json - let pkg_file = json!({ - "name": "test-package", - "version": "1.0.0", - "peerDependencies": { - "peer-pkg": "^3.0.0" - } - }); - - // Create a mock package-lock.json structure - let pkgs_in_pkg_lock = json!({ - "": { - "name": "test-package", - "version": "1.0.0", - "peerDependencies": { - "peer-pkg": "^3.0.0" - } - }, - "node_modules/peer-pkg": { - "name": "peer-pkg", - "version": "3.1.0", // Matches ^3.0.0 - "resolved": "https://registry.npmjs.org/peer-pkg/-/peer-pkg-3.1.0.tgz" - } - }); - - let invalid_deps = validate_deps(&pkg_file, &pkgs_in_pkg_lock).await.unwrap(); - // Valid peer dependency should not be treated as invalid - assert_eq!(invalid_deps.len(), 0); - } - - #[tokio::test] - async fn test_validate_deps_with_nested_dependencies() { - // Create a mock package.json - let pkg_file = json!({ - "name": "test-package", - "version": "1.0.0", - "dependencies": { - "parent-pkg": "^1.0.0" - } - }); - - // Create a mock package-lock.json structure with nested dependencies - let pkgs_in_pkg_lock = json!({ - "": { - "name": "test-package", - "version": "1.0.0", - "dependencies": { - "parent-pkg": "^1.0.0" - } - }, - "node_modules/parent-pkg": { - "name": "parent-pkg", - "version": "1.2.0", - "resolved": "https://registry.npmjs.org/parent-pkg/-/parent-pkg-1.2.0.tgz", - "dependencies": { - "nested-pkg": "^2.0.0" - } - }, - "node_modules/parent-pkg/node_modules/nested-pkg": { - "name": "nested-pkg", - "version": "2.1.0", // Matches ^2.0.0 - "resolved": "https://registry.npmjs.org/nested-pkg/-/nested-pkg-2.1.0.tgz" - } - }); - - let invalid_deps = validate_deps(&pkg_file, &pkgs_in_pkg_lock).await.unwrap(); - // All dependencies are valid - assert_eq!(invalid_deps.len(), 0); - } - #[test] fn test_package_struct_with_name_field() { // Test that Package struct can be deserialized with name field diff --git a/crates/pm/src/helper/mod.rs b/crates/pm/src/helper/mod.rs index 60ea4dca4..00137c470 100644 --- a/crates/pm/src/helper/mod.rs +++ b/crates/pm/src/helper/mod.rs @@ -3,6 +3,7 @@ pub mod cli; pub mod compatibility; pub mod deps; pub mod global_bin; +pub mod graph_builder; pub mod install_runtime; pub mod lock; pub mod package; diff --git a/crates/pm/src/helper/ruborist.rs b/crates/pm/src/helper/ruborist.rs index ca340d07e..c889ed096 100644 --- a/crates/pm/src/helper/ruborist.rs +++ b/crates/pm/src/helper/ruborist.rs @@ -1,337 +1,22 @@ -use std::collections::HashMap; use std::path::PathBuf; -use std::sync::Arc; use anyhow::{Context, Result}; use serde_json::Value; -use std::sync::Mutex; -use tokio::sync::Semaphore; +use crate::helper::graph_builder::build_deps; use crate::helper::install_runtime::install_runtime; use crate::helper::workspace::find_workspaces; -use crate::model::node::{Edge, EdgeType, Node}; +use crate::model::graph::{DependencyGraph, PackageNode}; +use crate::model::node::EdgeType; use crate::util::config::get_legacy_peer_deps; use crate::util::json::load_package_json_from_path; -use crate::util::logger::{PROGRESS_BAR, finish_progress_bar, log_progress, start_progress_bar}; -use crate::util::node_search::get_node_from_root_by_path; -use crate::util::registry::{ResolvedPackage, load_cache, resolve_dependency, store_cache}; -use crate::util::semver::matches; +use crate::util::logger::{finish_progress_bar, start_progress_bar}; +use crate::util::registry::{load_cache, store_cache}; +/// Ruborist - manages dependency graph building pub struct Ruborist { path: PathBuf, - pub ideal_tree: Option>, -} - -use once_cell::sync::Lazy; - -// concurrency limit default to 100 -static CONCURRENCY_LIMITER: Lazy> = Lazy::new(|| Arc::new(Semaphore::new(100))); - -pub async fn build_deps(root: Arc) -> Result<()> { - let legacy_peer_deps = get_legacy_peer_deps().await; - tracing::debug!("going to build deps for {root}, legacy_peer_deps: {legacy_peer_deps}"); - let current_level = Arc::new(Mutex::new(vec![root.clone()])); - // Track processed workspace nodes to prevent cycles - let processed_workspace_nodes = Arc::new(Mutex::new(std::collections::HashSet::new())); - - while !current_level.lock().unwrap().is_empty() { - let level_count = current_level.lock().unwrap().len(); - tracing::debug!("Starting new dependency level with {level_count} nodes"); - - let next_level = Arc::new(Mutex::new(Vec::new())); - let nodes = current_level.lock().unwrap().clone(); - let mut level_tasks = Vec::new(); - - for node in nodes { - let edges = node.edges_out.read().unwrap(); - let total_deps = edges.len(); - PROGRESS_BAR.inc_length(total_deps as u64); - log_progress(&format!("resolving {}", node.name)); - - let mut tasks = Vec::new(); - - for edge in edges.iter() { - let edge = edge.clone(); - let next_level = next_level.clone(); - let processed_workspace_nodes = processed_workspace_nodes.clone(); - - tasks.push(async move { - tracing::debug!("Task for {}@{} acquiring concurrency permit (available: {})", - edge.name, edge.spec, CONCURRENCY_LIMITER.available_permits()); - let _permit = CONCURRENCY_LIMITER.acquire().await.unwrap(); - tracing::debug!("Task for {}@{} acquired concurrency permit (remaining: {})", - edge.name, edge.spec, CONCURRENCY_LIMITER.available_permits()); - - if *edge.valid.read().unwrap() { - tracing::debug!("deps {}@{} already resolved", edge.name, edge.spec); - - // Only process workspace nodes from root to avoid cycles - if !edge.from.is_root() { - return Ok(()); - } - - // Add workspace node to next level if not processed before - if let Some(new_node) = edge.to.read().unwrap().as_ref().cloned() { - if !new_node.is_workspace() { - return Ok(()); - } - - let mut processed = processed_workspace_nodes.lock().unwrap(); - if !processed.contains(&new_node.name) { - processed.insert(new_node.name.clone()); - next_level.lock().unwrap().push(new_node); - } - } - - return Ok(()); - } - - tracing::debug!("going to build deps {}@{} from [{}]", edge.name, edge.spec, edge.from); - - // Add debug logs to track progress - let start_time = std::time::Instant::now(); - tracing::debug!("Starting dependency resolution for {}@{}", edge.name, edge.spec); - - match find_compatible_node(&edge.from, &edge.name, &edge.spec) { - FindResult::Reuse(existing_node) => { - tracing::debug!( - "resolved deps {}@{} => {} (reuse) took {:?}", - edge.name, &edge.spec, existing_node.version, start_time.elapsed() - ); - { - let mut to = edge.to.write().unwrap(); - *to = Some(existing_node.clone()); - let mut valid = edge.valid.write().unwrap(); - *valid = true; - } - - // update node type by edges - existing_node.add_invoke(&edge); - existing_node.update_type(); - } - FindResult::Conflict(conflict_node) => { - tracing::debug!("Conflict found for {}@{}, resolving...", edge.name, edge.spec); - let resolved = match resolve_dependency(&edge.name, &edge.spec, &edge.edge_type).await? { - Some(resolved) => { - tracing::debug!("Resolved dependency {}@{} => {}", edge.name, edge.spec, resolved.version); - resolved - }, - None => { - tracing::debug!("No resolution found for {}@{}", edge.name, edge.spec); - return Ok(()); - }, - }; - PROGRESS_BAR.inc(1); - tracing::debug!( - "resolved deps {}@{} => {} (conflict), need to fork, conflict_node: {}", - edge.name, &edge.spec, resolved.version, conflict_node - ); - // process conflict node - let install_parent = conflict_node; - let new_node = place_deps(edge.name.clone(), resolved, &install_parent) - .with_context(|| format!("Failed to place dependencies for {}@{} in conflict case", edge.name, edge.spec))?; - - - { - let mut parent = new_node.parent.write().unwrap(); - *parent = Some(install_parent.clone()); - let mut children = install_parent.children.write().unwrap(); - children.push(new_node.clone()); - - - let mut to = edge.to.write().unwrap(); - *to = Some(new_node.clone()); - let mut valid = edge.valid.write().unwrap(); - *valid = true; - // update node type - new_node.add_invoke(&edge); - new_node.update_type(); - } - - let dep_types = if legacy_peer_deps { - vec![ - ("dependencies", EdgeType::Prod), - ("optionalDependencies", EdgeType::Optional), - ] - } else { - vec![ - ("dependencies", EdgeType::Prod), - ("peerDependencies", EdgeType::Peer), - ("optionalDependencies", EdgeType::Optional), - ] - }; - - for (field, edge_type) in dep_types { - if let Some(deps) = new_node.package.get(field) - && let Some(deps) = deps.as_object() { - tracing::debug!("Processing {} dependencies for {}", field, new_node.name); - for (name, version) in deps { - let version_spec = version.as_str().unwrap_or("").to_string(); - let dep_edge = Edge::new(new_node.clone(), edge_type.clone(), name.clone(), version_spec); - tracing::debug!( - "add edge {}@{} for {}", - name, version, new_node.name - ); - new_node.add_edge(dep_edge).await; - } - tracing::debug!("Finished processing {} dependencies for {}", field, new_node.name); - } - } - - next_level.lock().unwrap().push(new_node); - } - FindResult::New(install_location) => { - tracing::debug!("New installation needed for {}@{}, resolving dependency...", edge.name, edge.spec); - let resolved = match resolve_dependency(&edge.name, &edge.spec, &edge.edge_type).await? { - Some(resolved) => { - tracing::debug!("Dependency resolved {}@{} => {}", edge.name, edge.spec, resolved.version); - resolved - }, - None => { - tracing::debug!("No resolution found for {}@{}", edge.name, edge.spec); - return Ok(()); - }, - }; - PROGRESS_BAR.inc(1); - tracing::debug!( - "resolved deps {}@{} => {} (new) took {:?}", - edge.name, &edge.spec, resolved.version, start_time.elapsed() - ); - let new_node = place_deps(edge.name.clone(), resolved, &install_location) - .with_context(|| format!("Failed to place dependencies for {}@{} in new case", edge.name, edge.spec))?; - let root_node = install_location.clone(); - - { - let mut parent = new_node.parent.write().unwrap(); - *parent = Some(root_node.clone()); - } - { - let mut children = root_node.children.write().unwrap(); - children.push(new_node.clone()); - } - { - let mut to = edge.to.write().unwrap(); - *to = Some(new_node.clone()); - let mut valid = edge.valid.write().unwrap(); - *valid = true; - // update node type - new_node.add_invoke(&edge); - new_node.update_type(); - } - - add_dependency_edge(&new_node, "dependencies", EdgeType::Prod).await; - - if !legacy_peer_deps { - add_dependency_edge(&new_node, "peerDependencies", EdgeType::Peer).await; - } - - add_dependency_edge(&new_node, "optionalDependencies", EdgeType::Optional).await; - - next_level.lock().unwrap().push(new_node); - } - } - - tracing::debug!("Task for {}@{} completed, releasing concurrency permit", edge.name, edge.spec); - Ok::<_, anyhow::Error>(()) - }); - } - level_tasks.push(futures::future::try_join_all(tasks)); - } - - // waiting for all tasks in this level to finish - let level_task_count = level_tasks.len(); - tracing::debug!("Waiting for {level_task_count} level tasks to complete"); - - futures::future::try_join_all(level_tasks) - .await - .map_err(|e| { - let err_msg = e - .chain() - .map(|err| format!(" {err}")) - .collect::>() - .join("\n"); - anyhow::anyhow!(err_msg) - })?; - - tracing::debug!("All {level_task_count} level tasks completed"); - - // continue to next level - *current_level.lock().unwrap() = next_level.lock().unwrap().clone(); - } - - Ok(()) -} - -// create a new node under parent -fn place_deps(name: String, pkg: ResolvedPackage, parent: &Arc) -> Result> { - let new_node = Node::new(name, parent.path.clone(), pkg.manifest); - - tracing::debug!( - "\nInstalling {}@{} under parent chain: {}", - new_node.name, - new_node.version, - parent - ); - // tracing::debug!(&print_parent_chain(parent)); - tracing::debug!(""); - - Ok(new_node) -} - -#[derive(Debug)] -pub enum FindResult { - Reuse(Arc), // can reuse - Conflict(Arc), // conflict, return parent node - New(Arc), // need to install under root node -} - -fn find_compatible_node(from: &Arc, name: &str, version_spec: &str) -> FindResult { - fn find_in_node( - node: &Arc, - name: &str, - version_spec: &str, - current: &Arc, - ) -> FindResult { - let children = node.children.read().unwrap(); - - for child in children.iter() { - if child.name == name { - if matches(version_spec, &child.version) { - tracing::debug!( - "found existing deps {}@{} got {}, place {}", - name, - version_spec, - child.version, - child - ); - return FindResult::Reuse(child.clone()); - } else { - tracing::debug!( - "found conflict deps {}@{} got {}, place {}", - name, - version_spec, - child.version, - child - ); - return FindResult::Conflict(current.clone()); - } - } - } - - // find in parent - if let Some(parent) = node.parent.read().unwrap().as_ref() { - find_in_node(parent, name, version_spec, current) - } else { - // not found, return new - FindResult::New(node.clone()) - } - } - - if let Some(parent) = from.parent.read().unwrap().as_ref() { - find_in_node(parent, name, version_spec, from) - } else { - find_in_node(from, name, version_spec, from) - } + pub ideal_tree: Option, } impl Ruborist { @@ -342,31 +27,30 @@ impl Ruborist { } } - async fn init_runtime(&mut self, root: Arc) -> Result<()> { - let deps = install_runtime(root.package.get("engines").unwrap_or(&Value::Null))?; + async fn init_runtime(&self, graph: &mut DependencyGraph) -> Result<()> { + let root_node = graph.get_node(graph.root_index).unwrap(); + let deps = install_runtime(root_node.package.get("engines").unwrap_or(&Value::Null))?; for (name, version) in deps { - let edge = Edge::new(root.clone(), EdgeType::Optional, name, version); - root.add_edge(edge).await; + graph.add_dependency_edge(graph.root_index, name, version, EdgeType::Optional); } Ok(()) } - async fn init_tree(&mut self) -> Result> { - // load package.json + async fn init_tree(&self) -> Result { + // Load package.json let pkg = load_package_json_from_path(&self.path).await?; - // create root node - let root = Node::new_root( - pkg["name"].as_str().unwrap_or("root").to_string(), - self.path.clone(), - pkg.clone(), - ); - tracing::debug!("root node: {root:?}"); + // Create dependency graph with root node + let mut graph = DependencyGraph::new(self.path.clone(), pkg.clone()); + tracing::debug!("root node created at {:?}", graph.root_index); + + // Initialize runtime dependencies + self.init_runtime(&mut graph).await?; - self.init_runtime(root.clone()).await?; - self.init_workspaces(root.clone()).await?; + // Initialize workspaces + self.init_workspaces(&mut graph).await?; - // collect deps type + // Collect dependency types let legacy_peer_deps = get_legacy_peer_deps().await; let dep_types = if legacy_peer_deps { vec![ @@ -383,33 +67,27 @@ impl Ruborist { ] }; - // process deps in root + // Add root dependencies for (field, dep_type) in dep_types { - if let Some(deps) = pkg.get(field) - && let Some(deps) = deps.as_object() - { + if let Some(deps) = pkg.get(field).and_then(|v| v.as_object()) { for (name, version) in deps { tracing::debug!("{name}: {version}"); let version_spec = version.as_str().unwrap_or("").to_string(); - - // create edge - let edge = Edge::new( - root.clone(), // need clone Arc - dep_type.clone(), + graph.add_dependency_edge( + graph.root_index, name.clone(), version_spec, + dep_type.clone(), ); - - tracing::debug!("add edge {}@{}", edge.name, edge.spec); - root.add_edge(edge).await; + tracing::debug!("add edge {}@{}", name, version); } } } - Ok(root) + Ok(graph) } - pub async fn init_workspaces(&mut self, root: Arc) -> Result<()> { + async fn init_workspaces(&self, graph: &mut DependencyGraph) -> Result<()> { let workspaces = find_workspaces(&self.path).await.map_err(|e| { let err_msg = e .chain() @@ -424,42 +102,29 @@ impl Ruborist { let version = pkg["version"].as_str().unwrap_or("").to_string(); // Create workspace node - let workspace_node = Node::new_workspace(name.clone(), path, pkg.clone()); + let workspace_node = + PackageNode::new_workspace(name.clone(), path.clone(), pkg.clone()); + let workspace_index = graph.add_node(workspace_node); // Create link node - let link_node = Node::new_link(name.clone(), workspace_node.clone()); - - // Create dependency edge - let dep_edge = Edge::new(root.clone(), EdgeType::Prod, name.clone(), version); - - // Set target node and validity for dependency edge - { - let mut valid = dep_edge.valid.write().unwrap(); - *valid = true; - - let mut to = dep_edge.to.write().unwrap(); - *to = Some(workspace_node.clone()); - } - - // Update parent relationships - { - let mut parent = workspace_node.parent.write().unwrap(); - *parent = Some(root.clone()); - } - { - let mut parent = link_node.parent.write().unwrap(); - *parent = Some(root.clone()); - } - { - let mut children = root.children.write().unwrap(); - children.push(workspace_node.clone()); - children.push(link_node); - } - - // Add dependency edge - root.add_edge(dep_edge).await; + let link_node = + PackageNode::new_link(name.clone(), path.clone(), pkg.clone(), version.clone()); + let link_index = graph.add_node(link_node); + + // Add physical edges + graph.add_physical_edge(graph.root_index, workspace_index); + graph.add_physical_edge(graph.root_index, link_index); + + // Create and mark dependency edge as resolved + let dep_edge_id = graph.add_dependency_edge( + graph.root_index, + name.clone(), + version.clone(), + EdgeType::Prod, + ); + graph.mark_dependency_resolved(dep_edge_id, workspace_index); - tracing::debug!("Added workspace: {} {:?}", name, workspace_node.path); + tracing::debug!("Added workspace: {} {:?}", name, path); // Process workspace dependencies let legacy_peer_deps = get_legacy_peer_deps().await; @@ -479,24 +144,16 @@ impl Ruborist { }; for (field, edge_type) in dep_types { - if let Some(deps) = workspace_node.package.get(field) - && let Some(deps) = deps.as_object() - { - for (name, version) in deps { + if let Some(deps) = pkg.get(field).and_then(|v| v.as_object()) { + for (dep_name, version) in deps { let version_spec = version.as_str().unwrap_or("").to_string(); - let dep_edge = Edge::new( - workspace_node.clone(), - edge_type.clone(), - name.clone(), + graph.add_dependency_edge( + workspace_index, + dep_name.clone(), version_spec, + edge_type.clone(), ); - tracing::debug!( - "add edge {}@{} for {}", - name, - version, - workspace_node.name - ); - workspace_node.add_edge(dep_edge).await; + tracing::debug!("add edge {}@{} for {}", dep_name, version, name); } } } @@ -505,398 +162,137 @@ impl Ruborist { Ok(()) } - pub async fn build_ideal_tree(&mut self) -> Result<()> { - let cache_path = self.path.join("./node_modules/.utoo-manifest.json"); - load_cache(&cache_path) - .await - .context("Failed to load cache")?; - let root = self.init_tree().await?; - - start_progress_bar(); - build_deps(root.clone()).await?; - - let res = self.get_dup_deps(root.clone()); - for dup_node in res { - self.replace_deps(dup_node).await?; - } - - store_cache(&cache_path) - .await - .context("Failed to store cache")?; - finish_progress_bar("package-lock.json resolved"); - self.ideal_tree = Some(root); - Ok(()) - } - + /// Build workspace tree (only resolves workspace dependencies, not external packages) pub async fn build_workspace_tree(&mut self) -> Result<()> { - let root = self.init_tree().await?; + let mut graph = self.init_tree().await?; start_progress_bar(); - // init_tree has already loaded all workspace - let children = root.children.read().unwrap(); - - // build a map of workspace nodes - let mut workspace_map = HashMap::new(); - for workspace in children.iter() { - if workspace.is_workspace() { - workspace_map.insert(workspace.name.clone(), workspace.clone()); + // Build a map of workspace nodes for quick lookup + let mut workspace_map = std::collections::HashMap::new(); + for node_idx in graph.graph.node_indices() { + let node = graph.get_node(node_idx).unwrap(); + if node.is_workspace() { + workspace_map.insert(node.name.clone(), node_idx); } } - // find the deps between workspace - for workspace in children.iter() { - if workspace.is_link() { - continue; - } - PROGRESS_BAR.inc_length(1); - let edges = workspace.edges_out.read().unwrap(); - for edge in edges.iter() { - if let Some(dep_workspace) = workspace_map.get(&edge.name) { - // find edges_out for workspace - let mut to = edge.to.write().unwrap(); - *to = Some(dep_workspace.clone()); - let mut valid = edge.valid.write().unwrap(); - *valid = true; - - tracing::debug!( - "Workspace dependency: {} -> {}", - workspace.name, - dep_workspace.name - ); - } - } - PROGRESS_BAR.inc(1); - } - - finish_progress_bar("workspace resolved"); - self.ideal_tree = Some(root.clone()); - Ok(()) - } - - pub fn get_dup_deps(&self, root: Arc) -> Vec> { - let mut duplicates = Vec::new(); + // Collect all workspace dependency resolutions first + let mut resolutions = Vec::new(); - fn process_node(node: &Arc, duplicates: &mut Vec>) { - let children = node.children.read().unwrap(); - let mut name_map: HashMap>> = HashMap::new(); + for node_idx in graph.graph.node_indices() { + // Check if this is a workspace node (not link) + let (is_link, is_workspace, node_name) = { + let node = graph.get_node(node_idx).unwrap(); + (node.is_link(), node.is_workspace(), node.name.clone()) + }; - // find duplicate deps - for child in children.iter() { - if child.is_workspace() { - continue; - } - name_map - .entry(child.name.clone()) - .or_default() - .push(child.clone()); + if is_link || !is_workspace { + continue; } - // handle dup node - for (_, nodes) in name_map { - if nodes.len() > 1 { - // find max edges_in node to save the cost - let mut max_edges = 0; - let mut primary_node = None; - - for node in &nodes { - let edges_count = node.edges_in.read().unwrap().len(); - if edges_count > max_edges { - max_edges = edges_count; - primary_node = Some(node.clone()); - } - } - - // add to duplicates - if let Some(primary) = primary_node { - for node in nodes { - if !Arc::ptr_eq(&node, &primary) { - duplicates.push(node); - } - } - } + // Get dependency edges for this workspace + let dep_edges: Vec<_> = graph.get_dependency_edges(node_idx); + + for (edge_id, dep_edge) in dep_edges { + // Check if this dependency is another workspace + if let Some(&dep_workspace_idx) = workspace_map.get(&dep_edge.name) { + resolutions.push(( + edge_id, + dep_workspace_idx, + node_name.clone(), + dep_edge.name.clone(), + )); } } - - for child in children.iter() { - process_node(child, duplicates); - } } - process_node(&root, &mut duplicates); - duplicates - } + // Now apply all resolutions + for (edge_id, dep_workspace_idx, from_name, to_name) in resolutions { + graph.mark_dependency_resolved(edge_id, dep_workspace_idx); - pub async fn replace_deps(&self, node: Arc) -> Result<()> { - tracing::debug!("going to replace node {node}"); - // 1. remove from parent node - if let Some(parent) = node.parent.read().unwrap().as_ref() { - let mut parent_children = parent.children.write().unwrap(); - parent_children.retain(|child| !Arc::ptr_eq(child, &node)); - } - - // 2. clean edges_out - { - let mut edges_out = node.edges_out.write().unwrap(); - edges_out.clear(); - } - - // 3. clean edges_in - let edges_from = { - let edges_in = node.edges_in.read().unwrap(); - edges_in - .iter() - .map(|edge| { - let mut valid = edge.valid.write().unwrap(); - *valid = false; - - let mut to = edge.to.write().unwrap(); - *to = None; - - edge.from.clone() - }) - .collect::>() - }; - - // 4. rebuild deps - for from_node in edges_from { - build_deps(from_node).await?; + tracing::debug!("Workspace dependency: {} -> {}", from_name, to_name); } + finish_progress_bar("workspace resolved"); + self.ideal_tree = Some(graph); Ok(()) } - pub async fn fix_dep_path(&self, pkg_path: &str, pkg_name: &str) -> Result<()> { - let root = self - .ideal_tree - .as_ref() - .ok_or_else(|| anyhow::anyhow!("Ideal tree not initialized"))?; - - let current_node = get_node_from_root_by_path(root, pkg_path).await?; - - // Now we have the target node, find and fix the dependency - let edges_to_fix = { - let edges = current_node.edges_out.read().unwrap(); - edges - .iter() - .filter(|edge| edge.name == pkg_name) - .cloned() - .collect::>() - }; + pub async fn build_ideal_tree(&mut self) -> Result<()> { + let cache_path = self.path.join("./node_modules/.utoo-manifest.json"); + load_cache(&cache_path) + .await + .context("Failed to load cache")?; - for edge in edges_to_fix { - let to_node = { - let to_guard = edge.to.read().unwrap(); - to_guard.as_ref().unwrap().clone() - }; - tracing::debug!( - "Fixing dependency: {}, from: {}, to: {}", - edge.name, - edge.from, - to_node - ); - *edge.valid.write().unwrap() = false; - build_deps(current_node.clone()).await?; - } + let mut graph = self.init_tree().await?; - Ok(()) - } -} - -async fn add_dependency_edge(node: &Arc, field: &str, edge_type: EdgeType) { - if let Some(deps) = node.package.get(field) - && let Some(deps) = deps.as_object() - { - for (name, version) in deps { - let version_spec = version.as_str().unwrap_or("").to_string(); - let dep_edge = Edge::new(node.clone(), edge_type.clone(), name.clone(), version_spec); - tracing::debug!("add edge {}@{} for {}", name, version, node.name); - node.add_edge(dep_edge).await; - } - } -} + // Check if project cache exists + let project_cache_path = self.path.join("node_modules/.utoo-manifest.json"); + let has_project_cache = tokio::fs::try_exists(&project_cache_path) + .await + .unwrap_or(false); -#[cfg(test)] -mod tests { - use serde_json::json; - use std::path::PathBuf; - - use super::*; - use crate::model::node::Node; - - #[tokio::test] - async fn test_fix_dep_path() { - // Create a mock root node - let root = Node::new_root( - "test-package".to_string(), - PathBuf::from("."), - json!({ - "name": "test-package", - "version": "1.0.0", - "dependencies": { - "lodash": "^4.17.20" - } - }), - ); - - // Create a child node - let child = Node::new( - "lodash".to_string(), - PathBuf::from("node_modules/lodash"), - json!({ - "name": "lodash", - "version": "4.17.20" - }), - ); - - // Add child to root - { - let mut children = root.children.write().unwrap(); - children.push(child.clone()); - } + // Only preload if project cache doesn't exist + if !has_project_cache { + let legacy_peer_deps = get_legacy_peer_deps().await; + let mut all_deps = std::collections::HashSet::new(); - // Create Ruborist instance - let mut ruborist = Ruborist::new(PathBuf::from(".")); - ruborist.ideal_tree = Some(root.clone()); + // Collect root dependencies + let root_node = graph.get_node(graph.root_index).unwrap(); + let root_deps = crate::service::preload::PreloadService::collect_root_dependencies( + &root_node.package, + legacy_peer_deps, + ); + for dep in root_deps { + all_deps.insert(dep); + } - // Test fixing dependency path - let result = ruborist.fix_dep_path("", "lodash").await; - assert!(result.is_ok()); - } + // Collect workspace dependencies + for node_index in graph.graph.node_indices() { + let node = graph.get_node(node_index).unwrap(); + if node.is_workspace() { + let workspace_deps = + crate::service::preload::PreloadService::collect_root_dependencies( + &node.package, + legacy_peer_deps, + ); + for dep in workspace_deps { + all_deps.insert(dep); + } + } + } - #[tokio::test] - async fn test_fix_dep_path_with_invalid_path() { - // Create a mock root node - let root = Node::new_root( - "test-package".to_string(), - PathBuf::from("."), - json!({ - "name": "test-package", - "version": "1.0.0" - }), - ); - - // Create Ruborist instance - let mut ruborist = Ruborist::new(PathBuf::from(".")); - ruborist.ideal_tree = Some(root.clone()); - - // Test fixing non-existent dependency path - let result = ruborist.fix_dep_path("invalid/path", "lodash").await; - assert!(result.is_err()); - } + let initial_deps: Vec<_> = all_deps.into_iter().collect(); - #[tokio::test] - async fn test_fix_dep_path_without_ideal_tree() { - // Create Ruborist instance without ideal tree - let ruborist = Ruborist::new(PathBuf::from(".")); + tracing::info!( + "No project cache found, preloading {} dependencies (root + workspaces)", + initial_deps.len() + ); + if let Err(e) = + crate::service::preload::PreloadService::preload(initial_deps, true).await + { + tracing::warn!("Preload failed, continuing with normal resolution: {}", e); + } + } else { + tracing::info!( + "Project cache found at {}, skipping preload", + project_cache_path.display() + ); + } - // Test fixing dependency path without ideal tree - let result = ruborist.fix_dep_path("", "lodash").await; - assert!(result.is_err()); - } + start_progress_bar(); + build_deps(&mut graph).await?; - #[tokio::test] - async fn test_build_deps_with_workspace_cycle() { - // Create a mock root node - let root = Node::new_root( - "test-monorepo".to_string(), - PathBuf::from("."), - json!({ - "name": "test-monorepo", - "version": "1.0.0", - "workspaces": ["packages/*"] - }), - ); - - // Create workspace A - let workspace_a = Node::new_workspace( - "workspace-a".to_string(), - PathBuf::from("packages/workspace-a"), - json!({ - "name": "workspace-a", - "version": "1.0.0", - "dependencies": { - "workspace-b": "1.0.0" - } - }), - ); - - // Create workspace B - let workspace_b = Node::new_workspace( - "workspace-b".to_string(), - PathBuf::from("packages/workspace-b"), - json!({ - "name": "workspace-b", - "version": "1.0.0", - "dependencies": { - "workspace-a": "1.0.0" - } - }), - ); - - // Add workspaces to root - { - let mut children = root.children.write().unwrap(); - children.push(workspace_a.clone()); - children.push(workspace_b.clone()); - } + // TODO: Implement dedup_deps for graph + // self.dedup_deps(&mut graph).await?; - // Create dependency edges - let edge_a_to_b = Edge::new( - workspace_a.clone(), - EdgeType::Prod, - "workspace-b".to_string(), - "1.0.0".to_string(), - ); - let edge_b_to_a = Edge::new( - workspace_b.clone(), - EdgeType::Prod, - "workspace-a".to_string(), - "1.0.0".to_string(), - ); - - // Add edges to workspaces - workspace_a.add_edge(edge_a_to_b).await; - workspace_b.add_edge(edge_b_to_a).await; - - // Test build_deps - let result = build_deps(root.clone()).await; - assert!( - result.is_ok(), - "build_deps should handle workspace cycles successfully" - ); - - // Verify that both workspaces are processed exactly once - let mut processed_workspaces = std::collections::HashSet::new(); - let mut stack = vec![root]; - - while let Some(node) = stack.pop() { - let children = node.children.read().unwrap(); - for child in children.iter() { - if child.is_workspace() { - assert!( - !processed_workspaces.contains(&child.name), - "Workspace {} should only be processed once", - child.name - ); - processed_workspaces.insert(child.name.clone()); - } - stack.push(child.clone()); - } - } + store_cache(&cache_path) + .await + .context("Failed to store cache")?; + finish_progress_bar("package-lock.json resolved"); - assert_eq!( - processed_workspaces.len(), - 2, - "Should process exactly 2 workspaces" - ); - assert!( - processed_workspaces.contains("workspace-a"), - "Should process workspace-a" - ); - assert!( - processed_workspaces.contains("workspace-b"), - "Should process workspace-b" - ); + self.ideal_tree = Some(graph); + Ok(()) } } diff --git a/crates/pm/src/model/graph.rs b/crates/pm/src/model/graph.rs new file mode 100644 index 000000000..f1ebc04f3 --- /dev/null +++ b/crates/pm/src/model/graph.rs @@ -0,0 +1,880 @@ +use petgraph::Direction::{Incoming, Outgoing}; +use petgraph::graph::{DiGraph, EdgeIndex, NodeIndex}; +use petgraph::visit::EdgeRef; +use serde_json::{Value, json}; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +use super::node::{EdgeType, NodeType}; +use super::override_rule::Overrides; +use crate::util::semver::matches; + +/// Package node in the dependency graph +#[derive(Debug, Clone)] +pub struct PackageNode { + pub path: PathBuf, + pub name: String, + pub version: String, + pub package: Value, + pub node_type: NodeType, + pub is_prod: bool, + pub is_dev: bool, + pub is_peer: bool, + pub is_optional: bool, +} + +impl PackageNode { + pub fn new(name: String, path: PathBuf, package: Value) -> Self { + Self { + path, + name, + version: package["version"].as_str().unwrap_or("").to_string(), + package, + node_type: NodeType::Regular, + is_prod: false, + is_dev: false, + is_peer: false, + is_optional: false, + } + } + + pub fn new_root(name: String, path: PathBuf, package: Value) -> Self { + Self { + path, + name, + version: package["version"].as_str().unwrap_or("").to_string(), + package, + node_type: NodeType::Root, + is_prod: false, + is_dev: false, + is_peer: false, + is_optional: false, + } + } + + pub fn new_workspace(name: String, path: PathBuf, package: Value) -> Self { + Self { + path, + name, + version: package["version"].as_str().unwrap_or("*").to_string(), + package, + node_type: NodeType::Workspace, + is_prod: false, + is_dev: false, + is_peer: false, + is_optional: false, + } + } + + pub fn new_link(name: String, path: PathBuf, package: Value, version: String) -> Self { + Self { + path, + name, + version, + package, + node_type: NodeType::Link, + is_prod: false, + is_dev: false, + is_peer: false, + is_optional: false, + } + } + + pub fn is_root(&self) -> bool { + self.node_type == NodeType::Root + } + + pub fn is_workspace(&self) -> bool { + self.node_type == NodeType::Workspace + } + + pub fn is_link(&self) -> bool { + self.node_type == NodeType::Link + } +} + +/// Edge in the dependency graph +#[derive(Debug, Clone)] +pub enum GraphEdge { + Physical, + Dependency(DependencyEdge), +} + +/// Dependency edge data +#[derive(Debug, Clone)] +pub struct DependencyEdge { + pub name: String, + pub spec: String, + pub edge_type: EdgeType, + pub valid: bool, + pub to: Option, +} + +impl DependencyEdge { + pub fn new(name: String, spec: String, edge_type: EdgeType) -> Self { + Self { + name, + spec: if spec.trim().is_empty() { + "*".to_string() + } else { + spec + }, + edge_type, + valid: false, + to: None, + } + } +} + +/// The main dependency graph structure +pub struct DependencyGraph { + pub graph: DiGraph, + pub root_index: NodeIndex, + // Project-level overrides configuration + overrides: Option, + // Fast lookup set for override names + override_names: std::collections::HashSet, +} + +impl DependencyGraph { + /// Create a new dependency graph with a root node + pub fn new(path: PathBuf, package: Value) -> Self { + let mut graph = DiGraph::new(); + let name = package["name"].as_str().unwrap_or("root").to_string(); + + // Parse overrides from package.json + let overrides = Overrides::parse(package.clone()); + + // Extract override names for fast lookup + let override_names = if let Some(ref rules) = overrides { + let names: std::collections::HashSet = + rules.rules.iter().map(|r| r.name.clone()).collect(); + tracing::debug!("Parsed {} override rules", rules.rules.len()); + for rule in &rules.rules { + tracing::debug!( + " Rule: {}@{} -> {}, parent: {:?}", + rule.name, + rule.spec, + rule.target_spec, + rule.parent + .as_ref() + .map(|p| format!("{}@{}", p.name, p.spec)) + ); + } + names + } else { + std::collections::HashSet::new() + }; + + let root_node = PackageNode::new_root(name, path, package); + let root_index = graph.add_node(root_node); + + Self { + graph, + root_index, + overrides, + override_names, + } + } + + /// Add a package node to the graph + pub fn add_node(&mut self, node: PackageNode) -> NodeIndex { + self.graph.add_node(node) + } + + /// Add a physical parent-child edge + pub fn add_physical_edge(&mut self, parent: NodeIndex, child: NodeIndex) -> EdgeIndex { + self.graph.add_edge(parent, child, GraphEdge::Physical) + } + + /// Add a dependency edge + pub fn add_dependency_edge( + &mut self, + from: NodeIndex, + name: String, + spec: String, + edge_type: EdgeType, + ) -> EdgeIndex { + let dep_edge = DependencyEdge::new(name, spec, edge_type); + self.graph + .add_edge(from, from, GraphEdge::Dependency(dep_edge)) + } + + /// Get the physical parent of a node + pub fn get_physical_parent(&self, node: NodeIndex) -> Option { + self.graph + .edges_directed(node, Incoming) + .find(|edge| matches!(edge.weight(), GraphEdge::Physical)) + .map(|edge| edge.source()) + } + + /// Get all physical children of a node + pub fn get_physical_children(&self, node: NodeIndex) -> Vec { + self.graph + .edges_directed(node, Outgoing) + .filter(|edge| matches!(edge.weight(), GraphEdge::Physical)) + .map(|edge| edge.target()) + .collect() + } + + /// Get all dependency edges from a node + pub fn get_dependency_edges(&self, node: NodeIndex) -> Vec<(EdgeIndex, &DependencyEdge)> { + self.graph + .edges_directed(node, Outgoing) + .filter_map(|edge| { + if let GraphEdge::Dependency(dep) = edge.weight() { + Some((edge.id(), dep)) + } else { + None + } + }) + .collect() + } + + /// Mark a dependency edge as resolved + pub fn mark_dependency_resolved(&mut self, edge_id: EdgeIndex, target: NodeIndex) { + if let Some(GraphEdge::Dependency(dep)) = self.graph.edge_weight_mut(edge_id) { + dep.valid = true; + dep.to = Some(target); + } + } + + /// Get node by index + pub fn get_node(&self, index: NodeIndex) -> Option<&PackageNode> { + self.graph.node_weight(index) + } + + /// Get mutable node by index + pub fn get_node_mut(&mut self, index: NodeIndex) -> Option<&mut PackageNode> { + self.graph.node_weight_mut(index) + } + + /// Get all workspace nodes (excluding links) + pub fn get_workspace_nodes(&self) -> Vec { + self.graph + .node_indices() + .filter(|&idx| { + if let Some(node) = self.get_node(idx) { + node.is_workspace() && !node.is_link() + } else { + false + } + }) + .collect() + } + + /// Get all resolved dependency targets for a node + /// Returns Vec<(dependency_name, target_node_index)> + pub fn get_resolved_dependencies(&self, node_index: NodeIndex) -> Vec<(String, NodeIndex)> { + self.get_dependency_edges(node_index) + .into_iter() + .filter_map(|(_, dep)| { + if dep.valid { + dep.to.map(|target| (dep.name.clone(), target)) + } else { + None + } + }) + .collect() + } + + /// Serialize graph to package-lock.json format + pub fn serialize_to_packages(&self, root_path: &Path) -> (Value, i32) { + let mut packages = json!({}); + let mut stack = vec![(self.root_index, String::new())]; + let mut total_packages = 0; + + while let Some((node_index, prefix)) = stack.pop() { + let _node = &self.graph[node_index]; + + // Check for duplicate dependencies + self.check_duplicate_dependencies(node_index); + + // Create package info + let pkg_info = self.create_package_info(node_index, root_path, &mut total_packages); + + // Use empty string for root node + let key = if prefix.is_empty() { + String::new() + } else { + prefix.clone() + }; + packages[key] = pkg_info; + + // Add physical children to processing stack + for child_index in self.get_physical_children(node_index) { + let child = &self.graph[child_index]; + let child_prefix = if prefix.is_empty() { + if child.is_workspace() { + // Workspace nodes use relative path from root + child + .path + .strip_prefix(root_path) + .map(|p| p.to_string_lossy().to_string()) + .unwrap_or_else(|_| child.path.to_string_lossy().to_string()) + } else { + format!("node_modules/{}", child.name) + } + } else { + format!("{}/node_modules/{}", prefix, child.name) + }; + stack.push((child_index, child_prefix)); + } + } + + (packages, total_packages) + } + + /// Check for duplicate dependencies under a node and log warnings + fn check_duplicate_dependencies(&self, node_index: NodeIndex) { + let mut name_count = HashMap::new(); + for child_index in self.get_physical_children(node_index) { + let child = &self.graph[child_index]; + if !child.is_link() { + *name_count.entry(child.name.as_str()).or_insert(0) += 1; + } + } + for (name, count) in name_count { + if count > 1 { + let node = &self.graph[node_index]; + tracing::warn!( + "Found {} duplicate dependencies named '{}' under '{}'", + count, + name, + node.name + ); + } + } + } + + /// Create package information for a node + fn create_package_info( + &self, + node_index: NodeIndex, + root_path: &Path, + total_packages: &mut i32, + ) -> Value { + let node = &self.graph[node_index]; + + if node.is_root() { + // Root node: all fields are handled in create_root_package_info + self.create_root_package_info(node_index) + } else { + // Non-root nodes: create basic info then add package fields + let mut pkg_info = + self.create_non_root_package_info(node_index, root_path, total_packages); + self.add_package_fields(&mut pkg_info, node_index); + pkg_info + } + } + + /// Create package info for root node + fn create_root_package_info(&self, node_index: NodeIndex) -> Value { + let node = &self.graph[node_index]; + let mut info = json!({ + "name": node.name, + "version": node.version, + }); + + // Add optional fields from package.json + if let Some(engines) = node.package.get("engines") { + info["engines"] = engines.clone(); + } + if let Some(workspaces) = node.package.get("workspaces") { + info["workspaces"] = workspaces.clone(); + } + + // Add dependency fields directly from package.json (not from graph edges) + // This ensures workspace dependencies are not included + let dep_fields = vec![ + "dependencies", + "devDependencies", + "peerDependencies", + "optionalDependencies", + ]; + for field in dep_fields { + if let Some(deps) = node.package.get(field) + && deps.is_object() + && !deps.as_object().unwrap().is_empty() + { + info[field] = deps.clone(); + } + } + + info + } + + /// Create package info for non-root nodes + fn create_non_root_package_info( + &self, + node_index: NodeIndex, + root_path: &Path, + total_packages: &mut i32, + ) -> Value { + let node = &self.graph[node_index]; + let mut info = json!({ + "name": node.package.get("name"), + }); + + if node.is_workspace() { + info["version"] = json!(node.package.get("version")); + } else if node.is_link() { + info["link"] = json!(true); + let relative_path = self.get_relative_path(&node.path, root_path); + info["resolved"] = json!(relative_path); + } else { + // Regular package + *total_packages += 1; + info["version"] = json!(node.package.get("version")); + + // Get resolved and integrity from dist field + let empty_dist = json!(""); + let dist = node.package.get("dist").unwrap_or(&empty_dist); + info["resolved"] = json!(dist.get("tarball")); + info["integrity"] = json!(dist.get("integrity")); + } + + // Add optional flags (dev, optional, peer) + self.add_optional_flags(&mut info, node_index); + + info + } + + /// Add optional flags (peer, dev, optional, hasInstallScript) + fn add_optional_flags(&self, info: &mut Value, node_index: NodeIndex) { + let node = &self.graph[node_index]; + + if node.is_peer { + info["peer"] = json!(true); + } + + match (node.is_dev, node.is_optional) { + (true, true) => info["devOptional"] = json!(true), + (true, false) => info["dev"] = json!(true), + (false, true) => info["optional"] = json!(true), + _ => {} + } + + if node.package.get("hasInstallScript") == Some(&json!(true)) { + info["hasInstallScript"] = json!(true); + } + } + + /// Add package fields like dependencies, bin, license, etc. + fn add_package_fields(&self, pkg_info: &mut Value, node_index: NodeIndex) { + let node = &self.graph[node_index]; + + // Add various package.json fields + let fields = vec![ + "bin", + "license", + "engines", + "os", + "cpu", + "scripts", + "hasInstallScript", + ]; + + for field in fields { + if let Some(value) = node.package.get(field) { + pkg_info[field] = value.clone(); + } + } + + // Add dependency fields + self.add_dependency_fields(pkg_info, node_index); + } + + /// Add dependency fields to package info + fn add_dependency_fields(&self, pkg_info: &mut Value, node_index: NodeIndex) { + let dep_types = vec![ + ("dependencies", EdgeType::Prod), + ("devDependencies", EdgeType::Dev), + ("peerDependencies", EdgeType::Peer), + ("optionalDependencies", EdgeType::Optional), + ]; + + for (field_name, edge_type) in dep_types { + let deps = self.collect_dependencies(node_index, &edge_type); + if !deps.is_empty() { + pkg_info[field_name] = json!(deps); + } + } + } + + /// Collect dependencies of a specific type + fn collect_dependencies( + &self, + node_index: NodeIndex, + edge_type: &EdgeType, + ) -> HashMap { + let mut deps = HashMap::new(); + + for (_, dep_edge) in self.get_dependency_edges(node_index) { + if &dep_edge.edge_type == edge_type { + deps.insert(dep_edge.name.clone(), dep_edge.spec.clone()); + } + } + + deps + } + + /// Get relative path from root + fn get_relative_path(&self, path: &Path, root_path: &Path) -> String { + path.strip_prefix(root_path) + .unwrap_or(path) + .to_string_lossy() + .to_string() + } + + /// Find compatible node in parent chain for dependency resolution + pub fn find_compatible_node( + &self, + from: NodeIndex, + name: &str, + version_spec: &str, + ) -> FindResult { + // Check if this dependency has an override + let effective_spec = + if let Some(overridden_spec) = self.check_override(from, name, version_spec) { + tracing::debug!( + "Using override spec for {}@{} => {}", + name, + version_spec, + overridden_spec + ); + overridden_spec + } else { + version_spec.to_string() + }; + + // Get physical parent of from node + let parent = self.get_physical_parent(from).unwrap_or(from); + + // Recursively search up the parent chain with effective spec + self.find_in_parent_chain(parent, name, &effective_spec, from) + } + + /// Recursively search for compatible node in parent chain + fn find_in_parent_chain( + &self, + current: NodeIndex, + name: &str, + spec: &str, + requester: NodeIndex, + ) -> FindResult { + // Check all physical children of current node + for child_idx in self.get_physical_children(current) { + let child = &self.graph[child_idx]; + if child.name == name { + if matches(spec, &child.version) { + tracing::debug!( + "found existing deps {}@{} got {}, reuse at {:?}", + name, + spec, + child.version, + child_idx + ); + return FindResult::Reuse(child_idx); + } else { + tracing::debug!( + "found conflict deps {}@{} got {}, conflict at {:?}", + name, + spec, + child.version, + child_idx + ); + return FindResult::Conflict(requester); + } + } + } + + // Recurse to parent + if let Some(parent) = self.get_physical_parent(current) { + self.find_in_parent_chain(parent, name, spec, requester) + } else { + // Reached root, install here + FindResult::New(current) + } + } + + /// Collect the physical parent chain from a node up to root + /// Includes the 'from' node itself in the chain + /// Returns Vec<(name, version)> for version-aware matching + fn collect_parent_chain(&self, from: NodeIndex) -> Vec<(String, String)> { + let mut chain = Vec::new(); + let mut current = from; + + // First, add the current node itself (if not root) + let from_node = &self.graph[from]; + if !from_node.is_root() { + chain.push((from_node.name.clone(), from_node.version.clone())); + } + + // Then collect all physical parents up to root + while let Some(parent) = self.get_physical_parent(current) { + let parent_node = &self.graph[parent]; + if !parent_node.is_root() { + chain.push((parent_node.name.clone(), parent_node.version.clone())); + } + current = parent; + } + + chain.reverse(); + chain + } + + /// Check if an override rule applies and return the overridden spec + pub fn check_override(&self, from: NodeIndex, name: &str, spec: &str) -> Option { + // Fast path: skip if name is not in override_names + if !self.override_names.contains(name) { + return None; + } + + let overrides = self.overrides.as_ref()?; + + // Collect parent chain once + let parent_chain = self.collect_parent_chain(from); + + // Match override rules + for rule in &overrides.rules { + if rule.name != name { + continue; + } + + // Check if spec matches (handle wildcard) + if rule.spec != "*" && !matches(&rule.spec, spec) { + continue; + } + + // Check if parent chain matches + if self.matches_parent_chain_for_rule(rule, &parent_chain) { + tracing::debug!( + "Override matched: {}@{} => {} (from {:?}, chain: {:?})", + name, + spec, + rule.target_spec, + from, + parent_chain + ); + return Some(rule.target_spec.clone()); + } + } + + None + } + + /// Check if a parent chain matches an override rule's parent condition + /// Based on the original implementation in override_rule.rs + fn matches_parent_chain_for_rule( + &self, + rule: &crate::model::override_rule::OverrideRule, + parent_chain: &[(String, String)], + ) -> bool { + // If no parent requirement, always matches + if rule.parent.is_none() { + return true; + } + + let mut current_rule = rule.parent.as_ref(); + let mut parent_idx = 0; + + while let Some((parent_name, parent_version)) = parent_chain.get(parent_idx) { + if let Some(rule_ref) = current_rule + && parent_name == &rule_ref.name + { + // Check if version matches + let version_matches = if rule_ref.spec == "*" { + true + } else { + matches(&rule_ref.spec, parent_version) + }; + + if version_matches { + // Move to next parent requirement + if let Some(next_rule) = rule_ref.parent.as_ref() { + current_rule = Some(next_rule); + parent_idx += 1; + continue; + } else { + // All parent requirements matched + return true; + } + } + } + parent_idx += 1; + } + + // If current_rule is still Some, it means we didn't match all parent requirements + current_rule.is_none() + } +} + +/// Result of finding a compatible node +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum FindResult { + /// Can reuse existing node + Reuse(NodeIndex), + /// Conflict found, install under this parent + Conflict(NodeIndex), + /// Need to install under this parent (usually root) + New(NodeIndex), +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn test_create_graph() { + let pkg = json!({ + "name": "test", + "version": "1.0.0" + }); + let graph = DependencyGraph::new(PathBuf::from("."), pkg); + assert_eq!(graph.graph.node_count(), 1); + assert!(graph.graph[graph.root_index].is_root()); + } + + #[test] + fn test_add_nodes_and_edges() { + let pkg = json!({ + "name": "root", + "version": "1.0.0" + }); + let mut graph = DependencyGraph::new(PathBuf::from("."), pkg); + + let child_pkg = json!({ + "name": "lodash", + "version": "4.17.21" + }); + let child = PackageNode::new( + "lodash".to_string(), + PathBuf::from("node_modules/lodash"), + child_pkg, + ); + let child_idx = graph.add_node(child); + + graph.add_physical_edge(graph.root_index, child_idx); + + assert_eq!(graph.graph.node_count(), 2); + assert_eq!(graph.get_physical_children(graph.root_index).len(), 1); + assert_eq!(graph.get_physical_parent(child_idx), Some(graph.root_index)); + } + + #[test] + fn test_dependency_edges() { + let pkg = json!({ + "name": "root", + "version": "1.0.0" + }); + let mut graph = DependencyGraph::new(PathBuf::from("."), pkg); + + let edge_id = graph.add_dependency_edge( + graph.root_index, + "lodash".to_string(), + "^4.17.0".to_string(), + EdgeType::Prod, + ); + + let deps = graph.get_dependency_edges(graph.root_index); + assert_eq!(deps.len(), 1); + assert_eq!(deps[0].1.name, "lodash"); + assert_eq!(deps[0].1.spec, "^4.17.0"); + assert!(!deps[0].1.valid); + + // Mark as resolved + let child_pkg = json!({ + "name": "lodash", + "version": "4.17.21" + }); + let child = PackageNode::new( + "lodash".to_string(), + PathBuf::from("node_modules/lodash"), + child_pkg, + ); + let child_idx = graph.add_node(child); + graph.mark_dependency_resolved(edge_id, child_idx); + + let deps = graph.get_dependency_edges(graph.root_index); + assert!(deps[0].1.valid); + assert_eq!(deps[0].1.to, Some(child_idx)); + } + + #[test] + fn test_find_compatible_node_reuse() { + let pkg = json!({"name": "root", "version": "1.0.0"}); + let mut graph = DependencyGraph::new(PathBuf::from("."), pkg); + + // Add lodash@4.17.21 under root + let lodash = PackageNode::new( + "lodash".to_string(), + PathBuf::from("node_modules/lodash"), + json!({"name": "lodash", "version": "4.17.21"}), + ); + let lodash_idx = graph.add_node(lodash); + graph.add_physical_edge(graph.root_index, lodash_idx); + + // Should reuse existing lodash when spec matches + let result = graph.find_compatible_node(graph.root_index, "lodash", "^4.17.0"); + assert_eq!(result, FindResult::Reuse(lodash_idx)); + } + + #[test] + fn test_find_compatible_node_conflict() { + let pkg = json!({"name": "root", "version": "1.0.0"}); + let mut graph = DependencyGraph::new(PathBuf::from("."), pkg); + + // Add lodash@3.10.1 under root + let lodash = PackageNode::new( + "lodash".to_string(), + PathBuf::from("node_modules/lodash"), + json!({"name": "lodash", "version": "3.10.1"}), + ); + let lodash_idx = graph.add_node(lodash); + graph.add_physical_edge(graph.root_index, lodash_idx); + + // Should find conflict when spec doesn't match + let result = graph.find_compatible_node(graph.root_index, "lodash", "^4.17.0"); + assert_eq!(result, FindResult::Conflict(graph.root_index)); + } + + #[test] + fn test_find_compatible_node_new() { + let pkg = json!({"name": "root", "version": "1.0.0"}); + let graph = DependencyGraph::new(PathBuf::from("."), pkg); + + // Should return New when no existing node found + let result = graph.find_compatible_node(graph.root_index, "lodash", "^4.17.0"); + assert_eq!(result, FindResult::New(graph.root_index)); + } + + #[test] + fn test_find_compatible_node_nested() { + let pkg = json!({"name": "root", "version": "1.0.0"}); + let mut graph = DependencyGraph::new(PathBuf::from("."), pkg); + + // Add express under root + let express = PackageNode::new( + "express".to_string(), + PathBuf::from("node_modules/express"), + json!({"name": "express", "version": "4.18.0"}), + ); + let express_idx = graph.add_node(express); + graph.add_physical_edge(graph.root_index, express_idx); + + // Add lodash@4.17.21 under root + let lodash = PackageNode::new( + "lodash".to_string(), + PathBuf::from("node_modules/lodash"), + json!({"name": "lodash", "version": "4.17.21"}), + ); + let lodash_idx = graph.add_node(lodash); + graph.add_physical_edge(graph.root_index, lodash_idx); + + // From express, should find lodash in parent (root) + let result = graph.find_compatible_node(express_idx, "lodash", "^4.17.0"); + assert_eq!(result, FindResult::Reuse(lodash_idx)); + } +} diff --git a/crates/pm/src/model/mod.rs b/crates/pm/src/model/mod.rs index 1550d9fa2..9b7e3b3ca 100644 --- a/crates/pm/src/model/mod.rs +++ b/crates/pm/src/model/mod.rs @@ -1,3 +1,4 @@ +pub mod graph; pub mod manifest; pub mod node; pub mod override_rule; diff --git a/crates/pm/src/model/node.rs b/crates/pm/src/model/node.rs index 2f7ec895b..7c1635b3e 100644 --- a/crates/pm/src/model/node.rs +++ b/crates/pm/src/model/node.rs @@ -1,7 +1,4 @@ -use serde_json::Value; -use std::path::PathBuf; -use std::sync::{Arc, RwLock}; - +/// Edge type in dependency graph #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub enum EdgeType { Prod, // Production dependency @@ -10,6 +7,7 @@ pub enum EdgeType { Optional, // Optional dependency } +/// Node type in dependency graph #[derive(Debug, Clone, PartialEq)] pub enum NodeType { Root, @@ -17,321 +15,3 @@ pub enum NodeType { Workspace, Link, } - -#[derive(Debug)] -pub struct Node { - // Basic info (immutable) - pub name: String, - pub version: String, - pub path: PathBuf, - pub package: Value, - - // Nested relationships (need mutable access) - pub parent: RwLock>>, - pub children: RwLock>>, - - // Edge relationships (need mutable access) - pub edges_out: RwLock>>, - pub edges_in: RwLock>>, - - // Node type and flags (immutable) - pub node_type: NodeType, - pub target: RwLock>>, - - // Dependency type flags (mutable) - pub is_optional: RwLock>, - pub is_peer: RwLock>, - pub is_dev: RwLock>, - pub is_prod: RwLock>, - - // Overrides configuration - pub overrides: Option, -} - -#[derive(Debug)] -pub struct Edge { - // Basic info (immutable) - pub name: String, - pub spec: String, - - // Relationship info (immutable) - pub from: Arc, - pub to: RwLock>>, - - // Resolution status - pub valid: RwLock, - - // Edge type (immutable) - pub edge_type: EdgeType, -} - -impl Node { - pub fn new(name: String, path: PathBuf, pkg: Value) -> Arc { - Arc::new(Self { - name, - version: pkg["version"].as_str().unwrap_or("").to_string(), - path, - package: pkg, - parent: RwLock::new(None), - children: RwLock::new(Vec::new()), - edges_out: RwLock::new(Vec::new()), - edges_in: RwLock::new(Vec::new()), - node_type: NodeType::Regular, - target: RwLock::new(None), - is_dev: RwLock::new(None), - is_peer: RwLock::new(None), - is_optional: RwLock::new(None), - is_prod: RwLock::new(None), - overrides: None, - }) - } - - pub fn new_root(name: String, path: PathBuf, pkg: Value) -> Arc { - Arc::new(Self { - name, - version: pkg["version"].as_str().unwrap_or("").to_string(), - path, - package: pkg.clone(), - parent: RwLock::new(None), - children: RwLock::new(Vec::new()), - edges_out: RwLock::new(Vec::new()), - edges_in: RwLock::new(Vec::new()), - node_type: NodeType::Root, - target: RwLock::new(None), - is_dev: RwLock::new(None), - is_peer: RwLock::new(None), - is_optional: RwLock::new(None), - is_prod: RwLock::new(None), - overrides: super::override_rule::Overrides::parse(pkg), - }) - } - - pub fn new_link(name: String, target: Arc) -> Arc { - Arc::new(Self { - name, - path: target.path.clone(), - package: target.package.clone(), - version: target.version.clone(), - target: RwLock::new(Some(target)), - parent: RwLock::new(None), - children: RwLock::new(Vec::new()), - edges_out: RwLock::new(Vec::new()), - edges_in: RwLock::new(Vec::new()), - node_type: NodeType::Link, - is_dev: RwLock::new(None), - is_peer: RwLock::new(None), - is_optional: RwLock::new(None), - is_prod: RwLock::new(None), - overrides: None, - }) - } - - pub fn new_workspace(name: String, path: PathBuf, pkg: Value) -> Arc { - Arc::new(Self { - name, - version: pkg["version"].as_str().unwrap_or("*").to_string(), - path, - package: pkg, - parent: RwLock::new(None), - children: RwLock::new(Vec::new()), - edges_out: RwLock::new(Vec::new()), - edges_in: RwLock::new(Vec::new()), - node_type: NodeType::Workspace, - target: RwLock::new(None), - is_dev: RwLock::new(None), - is_peer: RwLock::new(None), - is_optional: RwLock::new(None), - is_prod: RwLock::new(None), - overrides: None, - }) - } - - pub fn is_root(&self) -> bool { - self.node_type == NodeType::Root - } - - pub fn is_workspace(&self) -> bool { - self.node_type == NodeType::Workspace - } - - pub fn is_link(&self) -> bool { - self.node_type == NodeType::Link - } - - // Add incoming edge reference - pub fn add_invoke(&self, edge: &Arc) { - let mut edges = self.edges_in.write().unwrap(); - edges.push(edge.clone()); - } - - pub async fn add_edge(&self, mut edge: Arc) { - // Find root node for override rules - let mut current = Some(edge.from.clone()); - let mut root = None; - - while let Some(node) = current { - if node.is_root() { - root = Some(node); - break; - } - current = node.parent.read().unwrap().as_ref().cloned(); - } - - // Apply override rules if exists - if let Some(root) = root - && let Some(overrides) = &root.overrides - { - // Collect parent chain information - let mut parent_chain = Vec::new(); - let mut current_node = edge.from.parent.read().unwrap().clone(); - - while let Some(node) = current_node { - parent_chain.push((node.name.clone(), node.version.clone())); - current_node = node.parent.read().unwrap().clone(); - } - - // Check each rule - for rule in &overrides.rules { - if overrides - .matches_rule(rule, &edge.name, &edge.spec, &parent_chain) - .await - { - if let Some(edge_mut) = Arc::get_mut(&mut edge) { - tracing::debug!( - "Override rule applied {}@{} => {}", - rule.name, - rule.spec, - rule.target_spec - ); - edge_mut.spec = rule.target_spec.clone(); - } - break; - } - } - } - - let mut edges = self.edges_out.write().unwrap(); - edges.push(edge); - } - - // Update node type based on incoming edges - pub fn update_type(&self) { - if self.is_root() { - return; - } - - let edges_in = self.edges_in.read().unwrap(); - if edges_in.is_empty() { - return; - } - - let mut has_prod = false; - let mut all_optional = true; - let mut all_dev = true; - let mut all_peer = true; - - // Analyze incoming edges - for edge in edges_in.iter() { - let from_node = &edge.from; - - if *from_node.is_prod.read().unwrap() == Some(true) && edge.edge_type == EdgeType::Prod - { - has_prod = true; - all_optional = false; - all_dev = false; - all_peer = false; - break; - } - - if *from_node.is_optional.read().unwrap() != Some(true) - && edge.edge_type != EdgeType::Optional - { - all_optional = false; - } - if *from_node.is_dev.read().unwrap() != Some(true) && edge.edge_type != EdgeType::Dev { - all_dev = false; - } - if *from_node.is_peer.read().unwrap() != Some(true) && edge.edge_type != EdgeType::Peer - { - all_peer = false; - } - } - - // Update node status - let mut changed = false; - - if has_prod { - if *self.is_prod.read().unwrap() != Some(true) { - *self.is_prod.write().unwrap() = Some(true); - *self.is_optional.write().unwrap() = Some(false); - *self.is_dev.write().unwrap() = Some(false); - *self.is_peer.write().unwrap() = Some(false); - changed = true; - } - } else if all_optional { - if *self.is_optional.read().unwrap() != Some(true) { - *self.is_optional.write().unwrap() = Some(true); - *self.is_prod.write().unwrap() = Some(false); - changed = true; - } - } else if all_dev { - if *self.is_dev.read().unwrap() != Some(true) { - *self.is_dev.write().unwrap() = Some(true); - *self.is_prod.write().unwrap() = Some(false); - changed = true; - } - } else if all_peer && *self.is_peer.read().unwrap() != Some(true) { - *self.is_peer.write().unwrap() = Some(true); - *self.is_prod.write().unwrap() = Some(false); - changed = true; - } - - // Propagate changes - if changed { - tracing::debug!( - "{}@{} type changed [all_optional {}]", - &self.name, - &self.version, - all_optional - ); - - let edges_out = self.edges_out.read().unwrap(); - for edge in edges_out.iter() { - if let Some(to_node) = edge.to.read().unwrap().as_ref() { - to_node.update_type(); - } - } - } - } -} - -impl std::fmt::Display for Node { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}@{}", self.name, self.version)?; - - if !self.is_root() - && let Some(parent) = self.parent.read().unwrap().as_ref() - { - write!(f, " <- {parent}")?; - } - - Ok(()) - } -} - -impl Edge { - pub fn new(from: Arc, edge_type: EdgeType, name: String, spec: String) -> Arc { - Arc::new(Self { - name, - spec: if spec.trim().is_empty() { - "*".to_string() - } else { - spec - }, - from, - to: RwLock::new(None), - valid: RwLock::new(false), - edge_type, - }) - } -} diff --git a/crates/pm/src/model/override_rule.rs b/crates/pm/src/model/override_rule.rs index 39ac50076..0b3bc21e7 100644 --- a/crates/pm/src/model/override_rule.rs +++ b/crates/pm/src/model/override_rule.rs @@ -1,7 +1,5 @@ use super::super::helper::package::parse_package_spec; use super::super::util::json::merge_json_objects; -use super::super::util::registry::resolve; -use super::super::util::semver::{is_valid_version, matches}; use serde_json::Value; #[derive(Debug, Clone)] @@ -12,7 +10,7 @@ pub struct OverrideRule { pub parent: Option>, } -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct Overrides { pub package: Value, pub rules: Vec, @@ -107,62 +105,4 @@ impl Overrides { _ => String::from("*"), } } - - pub async fn matches_rule( - &self, - rule: &OverrideRule, - dep_name: &str, - dep_version: &str, - parent_chain: &[(String, String)], - ) -> bool { - if rule.name != dep_name { - return false; - } - - if rule.spec != "*" { - let resolved_version = if is_valid_version(dep_version) { - dep_version.to_string() - } else { - match resolve(dep_name, dep_version).await { - Ok(pkg) => pkg.version, - Err(_) => return false, - } - }; - - if !matches(&rule.spec, &resolved_version) { - return false; - } - } - - if let Some(mut current_rule) = rule.parent.as_ref() { - let mut parent_idx = 0; - - while let Some((parent_name, parent_version)) = parent_chain.get(parent_idx) { - if parent_name == ¤t_rule.name { - let matches = if current_rule.spec == "*" { - true - } else { - matches(¤t_rule.spec, parent_version) - }; - - if matches { - if let Some(next_rule) = current_rule.parent.as_ref() { - current_rule = next_rule; - parent_idx += 1; - continue; - } else { - return true; - } - } - } - parent_idx += 1; - } - - if current_rule.parent.is_some() { - return false; - } - } - - true - } } diff --git a/crates/pm/src/service/dependency_resolution.rs b/crates/pm/src/service/dependency_resolution.rs index cdc07bb1b..f9e97b58a 100644 --- a/crates/pm/src/service/dependency_resolution.rs +++ b/crates/pm/src/service/dependency_resolution.rs @@ -1,12 +1,9 @@ use anyhow::{Context, Result}; use std::path::Path; -use crate::helper::lock::{ - PackageLock, build_ideal_tree_to_package_lock, serialize_tree_to_packages, -}; +use crate::helper::lock::{PackageLock, build_ideal_tree_to_package_lock}; use crate::helper::ruborist::Ruborist; use crate::service::workspace::WorkspaceService; -use crate::util::json::load_package_json_from_path; /// Dependency resolution service pub struct DependencyResolutionService; @@ -16,58 +13,13 @@ impl DependencyResolutionService { let mut ruborist = Ruborist::new(cwd); ruborist.build_ideal_tree().await?; - let pkg_file = load_package_json_from_path(cwd).await?; + let graph = ruborist.ideal_tree.as_ref().unwrap(); - const MAX_RETRIES: u32 = 5; - let mut retry_count = 0; + // Serialize graph to packages + let (_packages, _total) = graph.serialize_to_packages(cwd); - loop { - let (pkgs_in_tree, _) = { - let to_guard = ruborist.ideal_tree.as_ref().unwrap(); - serialize_tree_to_packages(to_guard, cwd) - }; - - let invalid_deps = Self::validate_deps(&pkg_file, &pkgs_in_tree).await?; - - if invalid_deps.is_empty() { - tracing::debug!("No invalid dependencies found"); - break; - } - - if retry_count >= MAX_RETRIES { - return Err(anyhow::anyhow!( - "Failed to fix dependencies after {MAX_RETRIES} retries" - )); - } - - for dep in invalid_deps { - tracing::debug!( - "Fixing dependency: {}/{}", - dep.package_path, - dep.dependency_name - ); - // Try to fix the dependency - if let Err(e) = ruborist - .fix_dep_path(&dep.package_path, &dep.dependency_name) - .await - { - tracing::debug!("Failed to fix dependency: {e}"); - return Err(anyhow::anyhow!("Failed to fix dependency: {e}")); - } else { - tracing::debug!( - "Fixed dependency: {}/{}", - dep.package_path, - dep.dependency_name - ); - } - } - - retry_count += 1; - } - - let tree = ruborist.ideal_tree.unwrap(); - // Return PackageLock directly, no disk write here - let package_lock = build_ideal_tree_to_package_lock(cwd, &tree).await?; + // Build package lock from graph + let package_lock = build_ideal_tree_to_package_lock(cwd, graph).await?; Ok(package_lock) } @@ -88,12 +40,4 @@ impl DependencyResolutionService { Ok(()) } - - pub async fn validate_deps( - pkg_file: &serde_json::Value, - pkgs_in_pkg_lock: &serde_json::Value, - ) -> Result> { - // Use the existing implementation from helper/lock.rs - crate::helper::lock::validate_deps(pkg_file, pkgs_in_pkg_lock).await - } } diff --git a/crates/pm/src/service/mod.rs b/crates/pm/src/service/mod.rs index d5bd14aaf..5db5f7597 100644 --- a/crates/pm/src/service/mod.rs +++ b/crates/pm/src/service/mod.rs @@ -8,6 +8,7 @@ pub mod http_client; pub mod install; pub mod package; pub mod package_management; +pub mod preload; pub mod rebuild; pub mod registry; pub mod script; diff --git a/crates/pm/src/service/preload.rs b/crates/pm/src/service/preload.rs new file mode 100644 index 000000000..d0bcccfaf --- /dev/null +++ b/crates/pm/src/service/preload.rs @@ -0,0 +1,520 @@ +use anyhow::Result; +use futures::stream::FuturesUnordered; +use futures::stream::StreamExt; +use std::collections::HashSet; +use std::sync::Arc; +use std::time::Duration; +use tokio::sync::Semaphore; + +use super::registry::RegistryService; +use crate::util::cache::get_cache_dir; +use crate::util::config::get_legacy_peer_deps; +use crate::util::downloader::download; + +/// Statistics for measuring request performance +#[derive(Debug, Default)] +struct RequestStats { + durations: Vec, +} + +impl RequestStats { + fn new() -> Self { + Self { + durations: Vec::new(), + } + } + + fn add(&mut self, duration: Duration) { + self.durations.push(duration); + } + + fn merge(&mut self, other: RequestStats) { + self.durations.extend(other.durations); + } + + fn get_percentile(&self, percentile: f64) -> Option { + if self.durations.is_empty() { + return None; + } + let mut sorted = self.durations.clone(); + sorted.sort(); + let index = ((sorted.len() as f64 * percentile / 100.0).ceil() as usize).saturating_sub(1); + sorted.get(index).copied() + } + + fn max(&self) -> Option { + self.durations.iter().max().copied() + } + + fn avg(&self) -> Option { + if self.durations.is_empty() { + return None; + } + let sum: Duration = self.durations.iter().sum(); + Some(sum / self.durations.len() as u32) + } +} + +/// Default concurrency for manifest fetching +const DEFAULT_MANIFEST_CONCURRENCY: usize = 64; + +/// Default concurrency for tarball downloading +const DEFAULT_DOWNLOAD_CONCURRENCY: usize = 100; + +/// Dependency item for preloading +/// Represents a package dependency that needs to be preloaded +#[derive(Debug, Clone)] +struct DepItem { + name: String, + spec: String, +} + +/// Result of preloading a single package +/// This structure improves readability compared to returning an 8-element tuple +#[derive(Debug)] +struct PreloadResult { + next_deps: Vec, + manifest_success: bool, + manifest_failed: bool, + download_success: bool, + download_failed: bool, + download_skipped: bool, + manifest_stats: RequestStats, + download_stats: RequestStats, +} + +/// Service for preloading package manifests to improve resolution speed +pub struct PreloadService; + +impl PreloadService { + /// Preload package manifests recursively with unordered concurrency + /// + /// # Arguments + /// * `initial_deps` - Initial list of (name, spec) pairs to preload + /// * `package_lock_only` - If true, only preload manifests; if false, also download tgz files + /// + /// # Returns + /// * `Ok(())` on success + pub async fn preload( + initial_deps: Vec<(String, String)>, + package_lock_only: bool, + ) -> Result<()> { + let start_time = std::time::Instant::now(); + + tracing::info!( + "Starting preload with {} initial dependencies, package_lock_only={}", + initial_deps.len(), + package_lock_only, + ); + + // Create semaphores for concurrency control + let manifest_semaphore = Arc::new(Semaphore::new(DEFAULT_MANIFEST_CONCURRENCY)); + let download_semaphore = Arc::new(Semaphore::new(DEFAULT_DOWNLOAD_CONCURRENCY)); + + // Track processed packages to avoid duplicates (use name@spec as key) + let mut processed = HashSet::new(); + + // Statistics + let mut manifest_success_count = 0usize; + let mut manifest_failed_count = 0usize; + let mut download_success_count = 0usize; + let mut download_failed_count = 0usize; + let mut download_skipped_count = 0usize; + let mut manifest_stats = RequestStats::new(); + let mut download_stats = RequestStats::new(); + + // Convert initial deps to DepItems + let mut queue: Vec = initial_deps + .into_iter() + .map(|(name, spec)| DepItem { name, spec }) + .collect(); + + let legacy_peer_deps = get_legacy_peer_deps().await; + + while !queue.is_empty() { + tracing::debug!("Processing batch of {} packages", queue.len()); + + let mut tasks = FuturesUnordered::new(); + + // Add tasks to FuturesUnordered + for item in queue.drain(..) { + let key = format!("{}@{}", item.name, item.spec); + + // Skip if already processed + if processed.contains(&key) { + continue; + } + processed.insert(key.clone()); + + let name = item.name.clone(); + let spec = item.spec.clone(); + let manifest_sem = Arc::clone(&manifest_semaphore); + let download_sem = Arc::clone(&download_semaphore); + + let task = async move { + // Acquire manifest permit + let _manifest_permit = manifest_sem.acquire().await.unwrap(); + + tracing::debug!("Preloading manifest for {}", key); + + let mut result = PreloadResult { + next_deps: Vec::new(), + manifest_success: false, + manifest_failed: false, + download_success: false, + download_failed: false, + download_skipped: false, + manifest_stats: RequestStats::new(), + download_stats: RequestStats::new(), + }; + + // Measure manifest fetch time + let manifest_start = std::time::Instant::now(); + match RegistryService::resolve_package(&name, &spec).await { + Ok(resolved) => { + let manifest_duration = manifest_start.elapsed(); + result.manifest_stats.add(manifest_duration); + + tracing::debug!( + "Successfully preloaded manifest for {} => {} in {:.2}ms", + key, + resolved.version, + manifest_duration.as_secs_f64() * 1000.0 + ); + result.manifest_success = true; + + // Download tgz if needed + if !package_lock_only { + // Acquire download permit + let _download_permit = download_sem.acquire().await.unwrap(); + + let download_start = std::time::Instant::now(); + match Self::download_package( + &name, + &resolved.version, + &resolved.manifest, + ) + .await + { + Ok(was_cached) => { + let download_duration = download_start.elapsed(); + result.download_stats.add(download_duration); + + if was_cached { + result.download_skipped = true; + } else { + result.download_success = true; + } + } + Err(e) => { + // Preload download failure is not critical, just warn + tracing::warn!( + "Failed to download package {} during preload: {}", + key, + e + ); + result.download_failed = true; + } + } + } + + // Extract dependencies for next level + // Always pass false for include_dev - only root/workspace load devDeps + result.next_deps = Self::extract_dependencies( + &resolved.manifest, + false, // Don't load devDeps recursively + legacy_peer_deps, + ); + } + Err(e) => { + // Preload is best-effort, failure should not block + // Main build_deps will handle retries + tracing::debug!("Failed to preload manifest for {}: {}", key, e); + result.manifest_failed = true; + // result.next_deps is already an empty Vec from initialization + } + } + + result + }; + + tasks.push(task); + } + + // Collect results from all tasks + let mut next_queue = Vec::new(); + while let Some(result) = tasks.next().await { + // Collect statistics + if result.manifest_success { + manifest_success_count += 1; + } + if result.manifest_failed { + manifest_failed_count += 1; + } + if result.download_success { + download_success_count += 1; + } + if result.download_failed { + download_failed_count += 1; + } + if result.download_skipped { + download_skipped_count += 1; + } + + // Merge timing statistics + manifest_stats.merge(result.manifest_stats); + download_stats.merge(result.download_stats); + + // Collect deps for next level + next_queue.extend(result.next_deps); + } + + // Move to next level + queue = next_queue; + } + + let duration = start_time.elapsed(); + + tracing::info!( + "Preload completed in {:.2}s: {} packages processed | Manifests: {} success, {} failed | Downloads: {} success, {} failed, {} cached", + duration.as_secs_f64(), + processed.len(), + manifest_success_count, + manifest_failed_count, + download_success_count, + download_failed_count, + download_skipped_count + ); + + // Calculate and log success rates + let manifest_total = manifest_success_count + manifest_failed_count; + if manifest_total > 0 { + let manifest_success_rate = + (manifest_success_count as f64 / manifest_total as f64) * 100.0; + tracing::info!( + "Manifest success rate: {:.1}% ({}/{})", + manifest_success_rate, + manifest_success_count, + manifest_total + ); + } + + // Log manifest timing statistics + if let (Some(avg), Some(p50), Some(p90), Some(p99), Some(max)) = ( + manifest_stats.avg(), + manifest_stats.get_percentile(50.0), + manifest_stats.get_percentile(90.0), + manifest_stats.get_percentile(99.0), + manifest_stats.max(), + ) { + tracing::info!( + "Manifest fetch timing: avg={:.0}ms, p50={:.0}ms, p90={:.0}ms, p99={:.0}ms, max={:.0}ms", + avg.as_secs_f64() * 1000.0, + p50.as_secs_f64() * 1000.0, + p90.as_secs_f64() * 1000.0, + p99.as_secs_f64() * 1000.0, + max.as_secs_f64() * 1000.0 + ); + } + + if !package_lock_only { + let download_total = + download_success_count + download_failed_count + download_skipped_count; + if download_total > 0 { + let download_success_rate = ((download_success_count + download_skipped_count) + as f64 + / download_total as f64) + * 100.0; + tracing::info!( + "Download success rate: {:.1}% ({}/{})", + download_success_rate, + download_success_count + download_skipped_count, + download_total + ); + } + + // Log download timing statistics + if let (Some(avg), Some(p50), Some(p90), Some(max)) = ( + download_stats.avg(), + download_stats.get_percentile(50.0), + download_stats.get_percentile(90.0), + download_stats.max(), + ) { + tracing::info!( + "Download timing: avg={:.0}ms, p50={:.0}ms, p90={:.0}ms, max={:.0}ms", + avg.as_secs_f64() * 1000.0, + p50.as_secs_f64() * 1000.0, + p90.as_secs_f64() * 1000.0, + max.as_secs_f64() * 1000.0 + ); + } + } + + Ok(()) + } + + /// Download package tarball to cache + /// + /// Returns `Ok(true)` if the package was already cached, `Ok(false)` if it was downloaded + async fn download_package( + name: &str, + version: &str, + manifest: &serde_json::Value, + ) -> Result { + // Get tarball URL from manifest + let tarball_url = manifest["dist"]["tarball"] + .as_str() + .ok_or_else(|| anyhow::anyhow!("Failed to get tarball URL from manifest"))?; + + // Calculate cache path + let cache_dir = get_cache_dir(); + let cache_path = cache_dir.join(format!("{name}/{version}")); + let cache_flag_path = cache_dir.join(format!("{name}/{version}/_resolved")); + + // Download if not cached + if !tokio::fs::try_exists(&cache_flag_path).await? { + tracing::debug!("Downloading {} to {}", tarball_url, cache_path.display()); + download(tarball_url, &cache_path).await?; + Ok(false) // Downloaded + } else { + tracing::debug!("Package already cached: {}@{}", name, version); + Ok(true) // Cached + } + } + + /// Helper to extract dependencies from a specific field in package.json + fn extract_deps_from_field(manifest: &serde_json::Value, field: &str) -> Vec { + let mut deps = Vec::new(); + if let Some(dep_obj) = manifest.get(field).and_then(|v| v.as_object()) { + for (name, version) in dep_obj { + if let Some(version_str) = version.as_str() { + // Skip local dependencies (file:, link:, workspace:, portal:) + if version_str.starts_with("file:") + || version_str.starts_with("link:") + || version_str.starts_with("workspace:") + || version_str.starts_with("portal:") + { + tracing::debug!( + "Skipping local dependency in preload: {}@{}", + name, + version_str + ); + continue; + } + + deps.push(DepItem { + name: name.clone(), + spec: version_str.to_string(), + }); + } + } + } + deps + } + + /// Extract dependencies from a manifest + /// + /// # Arguments + /// * `manifest` - Package manifest JSON + /// * `include_dev` - Whether to include devDependencies (true for root/workspace) + /// * `legacy_peer_deps` - Whether to skip peerDependencies + fn extract_dependencies( + manifest: &serde_json::Value, + include_dev: bool, + legacy_peer_deps: bool, + ) -> Vec { + let mut deps = Vec::new(); + + // Always include prod dependencies + deps.extend(Self::extract_deps_from_field(manifest, "dependencies")); + + // Include dev deps for root/workspace nodes + if include_dev { + deps.extend(Self::extract_deps_from_field(manifest, "devDependencies")); + } + + // Include peer deps unless legacy mode + if !legacy_peer_deps { + deps.extend(Self::extract_deps_from_field(manifest, "peerDependencies")); + } + + // Always include optional dependencies + deps.extend(Self::extract_deps_from_field( + manifest, + "optionalDependencies", + )); + + deps + } + + /// Collect all dependencies from root package manifest (including devDependencies) + pub fn collect_root_dependencies( + manifest: &serde_json::Value, + legacy_peer_deps: bool, + ) -> Vec<(String, String)> { + let items = Self::extract_dependencies(manifest, true, legacy_peer_deps); + items + .into_iter() + .map(|item| (item.name, item.spec)) + .collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + #[test] + fn test_extract_dependencies_with_dev() { + let manifest = json!({ + "name": "test-package", + "version": "1.0.0", + "dependencies": { + "lodash": "^4.17.21", + "react": "^18.0.0" + }, + "devDependencies": { + "jest": "^29.0.0" + }, + "peerDependencies": { + "typescript": "^5.0.0" + }, + "optionalDependencies": { + "fsevents": "^2.3.2" + } + }); + + // Root level: include dev, include peer (not legacy) + // Should include: lodash, react, jest, typescript, fsevents (5 total) + let deps = PreloadService::extract_dependencies(&manifest, true, false); + assert_eq!(deps.len(), 5); + + // Non-root level: exclude dev, include peer (not legacy) + // Should include: lodash, react, typescript, fsevents (4 total, no jest) + let deps = PreloadService::extract_dependencies(&manifest, false, false); + assert_eq!(deps.len(), 4); + + // Legacy mode: include dev, exclude peer + // Should include: lodash, react, jest, fsevents (4 total, no typescript) + let deps_legacy = PreloadService::extract_dependencies(&manifest, true, true); + assert_eq!(deps_legacy.len(), 4); + } + + #[test] + fn test_collect_root_dependencies() { + let manifest = json!({ + "dependencies": { + "lodash": "^4.17.21" + }, + "devDependencies": { + "jest": "^29.0.0" + } + }); + + let deps = PreloadService::collect_root_dependencies(&manifest, false); + assert_eq!(deps.len(), 2); + assert!(deps.contains(&("lodash".to_string(), "^4.17.21".to_string()))); + assert!(deps.contains(&("jest".to_string(), "^29.0.0".to_string()))); + } +} diff --git a/crates/pm/src/service/workspace.rs b/crates/pm/src/service/workspace.rs index 63e98c4b2..f0864517b 100644 --- a/crates/pm/src/service/workspace.rs +++ b/crates/pm/src/service/workspace.rs @@ -23,18 +23,20 @@ impl WorkspaceService { let mut ruborist = Ruborist::new(cwd); ruborist.build_workspace_tree().await?; - if let Some(ideal_tree) = &ruborist.ideal_tree { + if let Some(graph) = &ruborist.ideal_tree { let mut node_list = Vec::new(); let mut node_json_list = Vec::new(); let mut edges = Vec::new(); let mut workspace_names = HashSet::new(); + // Get all workspace nodes (excluding links) + let workspace_nodes = graph.get_workspace_nodes(); + // Collect workspace nodes - for child in ideal_tree.children.read().unwrap().iter() { - let name = child.name.clone(); - if child.is_link() { - continue; - } + for node_idx in &workspace_nodes { + let node = graph.get_node(*node_idx).unwrap(); + let name = node.name.clone(); + workspace_names.insert(name.clone()); // Create Node struct for the helper function @@ -43,19 +45,22 @@ impl WorkspaceService { // Create JSON for output file node_json_list.push(json!({ "name": name, - "path": to_relative_path(&child.path, cwd), + "path": to_relative_path(&node.path, cwd), })); } - // Collect dependency edges - for child in ideal_tree.children.read().unwrap().iter() { - for edge in child.edges_out.read().unwrap().iter() { - if *edge.valid.read().unwrap() - && let Some(to_node) = edge.to.read().unwrap().as_ref() - { + // Collect dependency edges between workspaces + for node_idx in &workspace_nodes { + let node = graph.get_node(*node_idx).unwrap(); + let resolved_deps = graph.get_resolved_dependencies(*node_idx); + + for (_dep_name, target_idx) in resolved_deps { + let target_node = graph.get_node(target_idx).unwrap(); + // Only include workspace-to-workspace dependencies + if workspace_names.contains(&target_node.name) { // Create Edge struct: format is [to, from] meaning "to depends on from" - // So from=edge.from.name (dependency), to=to_node.name (dependent) - edges.push(Edge::new(to_node.name.clone(), edge.from.name.clone())); + // So from=dep_name (dependency), to=node.name (dependent) + edges.push(Edge::new(target_node.name.clone(), node.name.clone())); } } } diff --git a/crates/pm/src/util/cloner.rs b/crates/pm/src/util/cloner.rs index 427c21261..664488fb2 100644 --- a/crates/pm/src/util/cloner.rs +++ b/crates/pm/src/util/cloner.rs @@ -196,7 +196,7 @@ mod linux_clone { } else { // Try hardlink first for files in packages without install scripts if let Err(e) = fs::hard_link(&entry_path, &target_path).await { - tracing::debug!( + eprintln!( "Failed to create hardlink for file from {} to {}: {}, trying fast copy", entry_path.display(), target_path.display(), diff --git a/crates/pm/src/util/mod.rs b/crates/pm/src/util/mod.rs index 408489bd2..eb5ee092d 100644 --- a/crates/pm/src/util/mod.rs +++ b/crates/pm/src/util/mod.rs @@ -7,7 +7,6 @@ pub mod format_print; pub mod json; pub mod linker; pub mod logger; -pub mod node_search; pub mod registry; pub mod relative_path; pub mod retry; diff --git a/crates/pm/src/util/node_search.rs b/crates/pm/src/util/node_search.rs deleted file mode 100644 index 8b5720a5a..000000000 --- a/crates/pm/src/util/node_search.rs +++ /dev/null @@ -1,106 +0,0 @@ -use anyhow::Result; -use std::sync::Arc; - -use crate::model::node::Node; - -pub async fn get_node_from_root_by_path(root: &Arc, pkg_path: &str) -> Result> { - // Split the path by node_modules to get the package hierarchy - let path_parts: Vec<&str> = pkg_path - .trim_end_matches('/') // Remove trailing slash - .split("node_modules/") - .filter(|s| !s.is_empty()) - .map(|s| s.trim_end_matches('/')) // Remove trailing slash from each part - .collect(); - - // Start from the root node - let mut current_node = root.clone(); - - // Navigate through the node_modules hierarchy - for part in path_parts { - let mut found = false; - let next_node = { - let children = current_node.children.read().unwrap(); - let mut result = None; - - // Look for the next node in the hierarchy - for child in children.iter() { - if child.name == part { - result = Some(child.clone()); - found = true; - break; - } - } - result - }; - - if !found { - return Err(anyhow::anyhow!("Could not find package at path {pkg_path}")); - } - - current_node = next_node.unwrap(); - } - Ok(current_node) -} - -#[cfg(test)] -mod tests { - use super::*; - use serde_json::json; - use std::path::PathBuf; - - #[tokio::test] - async fn test_get_node_from_root_by_path() { - let root = Node::new_root( - "test-package".to_string(), - PathBuf::from("."), - json!({ - "name": "test-package", - "version": "1.0.0" - }), - ); - - let child1 = Node::new( - "lodash".to_string(), - PathBuf::from("node_modules/lodash"), - json!({ - "name": "lodash", - "version": "4.17.20" - }), - ); - - let child2 = Node::new( - "express".to_string(), - PathBuf::from("node_modules/express"), - json!({ - "name": "express", - "version": "4.17.1" - }), - ); - - { - let mut children = root.children.write().unwrap(); - children.push(child1.clone()); - children.push(child2.clone()); - } - - let result = get_node_from_root_by_path(&root, "node_modules/lodash").await; - assert!(result.is_ok()); - let node = result.unwrap(); - assert_eq!(node.name, "lodash"); - assert_eq!(node.version, "4.17.20"); - - let result = get_node_from_root_by_path(&root, "node_modules/express/").await; - assert!(result.is_ok()); - let node = result.unwrap(); - assert_eq!(node.name, "express"); - assert_eq!(node.version, "4.17.1"); - - let result = get_node_from_root_by_path(&root, "node_modules/non-existent").await; - assert!(result.is_err()); - - let result = get_node_from_root_by_path(&root, "").await; - assert!(result.is_ok()); - let node = result.unwrap(); - assert_eq!(node.name, "test-package"); - } -} diff --git a/crates/pm/src/util/semver.rs b/crates/pm/src/util/semver.rs index 107a3bf9f..3e111a4f8 100644 --- a/crates/pm/src/util/semver.rs +++ b/crates/pm/src/util/semver.rs @@ -34,10 +34,6 @@ pub fn matches(range: &str, version: &str) -> bool { req.matches(&version) } -pub fn is_valid_version(version: &str) -> bool { - Version::parse_from_npm(version).is_ok() -} - /// Find the maximum version from a list of versions that satisfies the given range pub fn max_satisfying<'a>(versions: impl Iterator, range: &str) -> Option { let req = VersionReq::parse_from_npm(range).ok()?; diff --git a/e2e/utoo-pm.sh b/e2e/utoo-pm.sh index c23f75b58..2dc1e8832 100755 --- a/e2e/utoo-pm.sh +++ b/e2e/utoo-pm.sh @@ -24,7 +24,6 @@ if [ ! -d "ant-design-x" ]; then fi cd ant-design-x echo "Installing dependencies for ant-design-x (next)..." -utoo deps || { echo -e "${RED}FAIL: utoo install failed for ant-design-x (next)${NC}"; exit 1; } utoo install --ignore-scripts || { echo -e "${RED}FAIL: utoo install failed for ant-design-x (next)${NC}"; exit 1; } utoo rebuild || { echo -e "${RED}FAIL: utoo install failed for ant-design-x (next)${NC}"; exit 1; } echo -e "${GREEN}PASS: ant-design-x (next) cloned and installed${NC}" @@ -37,8 +36,9 @@ if [ ! -d "ant-design" ]; then git clone --depth=1 --single-branch https://github.com/ant-design/ant-design.git fi cd ant-design +rm -rf ~/.cache/nm echo "Installing dependencies for ant-design..." -utoo install || { echo -e "${RED}FAIL: utoo install failed for ant-design${NC}"; exit 1; } +utoo install --ignore-scripts || { echo -e "${RED}FAIL: utoo install failed for ant-design${NC}"; exit 1; } echo -e "${GREEN}PASS: ant-design cloned and installed${NC}" cd ../../ From 14360d4fd78c191ef7e319cc238dae57b81b2cfc Mon Sep 17 00:00:00 2001 From: elrrrrrrr Date: Tue, 21 Oct 2025 21:20:57 +0800 Subject: [PATCH 2/3] fix: clone --- crates/pm/src/helper/graph_builder.rs | 87 ++++++++++++--------------- crates/pm/src/helper/ruborist.rs | 4 +- crates/pm/src/model/node.rs | 2 +- 3 files changed, 40 insertions(+), 53 deletions(-) diff --git a/crates/pm/src/helper/graph_builder.rs b/crates/pm/src/helper/graph_builder.rs index 9439090bb..4becf23af 100644 --- a/crates/pm/src/helper/graph_builder.rs +++ b/crates/pm/src/helper/graph_builder.rs @@ -7,7 +7,8 @@ use crate::util::config::get_legacy_peer_deps; use crate::util::logger::{PROGRESS_BAR, log_progress}; use crate::util::registry::{ResolvedPackage, resolve_dependency}; -/// Represents dependency edge information extracted from the graph +/// Represents UNRESOLVED dependency edge information extracted from the graph +/// Only used for edges that need to be resolved (is_valid = false) /// This structure improves readability compared to using tuples #[derive(Debug, Clone)] struct DependencyEdgeInfo { @@ -15,8 +16,6 @@ struct DependencyEdgeInfo { name: String, spec: String, edge_type: EdgeType, - is_valid: bool, - target: Option, } /// Represents node type information for dependency propagation @@ -29,7 +28,8 @@ struct NodeTypeInfo { is_optional: bool, } -/// Collect dependency edges from a node and convert to structured info +/// Collect ONLY unresolved dependency edges from a node +/// This optimization avoids cloning name/spec for already-resolved edges (workspace edges, etc.) /// This helper function improves code readability by avoiding complex tuple destructuring fn collect_dependency_edges( graph: &DependencyGraph, @@ -38,13 +38,12 @@ fn collect_dependency_edges( let dep_edges = graph.get_dependency_edges(node_index); dep_edges .into_iter() + .filter(|(_, dep)| !dep.valid) // Only collect unresolved edges .map(|(edge_id, dep)| DependencyEdgeInfo { edge_id, name: dep.name.clone(), spec: dep.spec.clone(), - edge_type: dep.edge_type.clone(), - is_valid: dep.valid, - target: dep.to, + edge_type: dep.edge_type, // EdgeType is Copy, no need to clone }) .collect() } @@ -63,35 +62,31 @@ pub async fn build_deps(graph: &mut DependencyGraph) -> Result<()> { let mut next_level = Vec::new(); for node_index in current_level { - // Collect all dependency edges from this node (both valid and invalid) - // We extract edge info into a dedicated struct to avoid borrow conflicts - // and improve code readability - let dependency_edges = collect_dependency_edges(graph, node_index); - - // Count unresolved edges for progress bar tracking - let unresolved_count = dependency_edges - .iter() - .filter(|edge| !edge.is_valid) - .count(); - PROGRESS_BAR.inc_length(unresolved_count as u64); + // Handle resolved edges first (no need to clone name/spec) + let resolved_edges: Vec<_> = graph + .get_dependency_edges(node_index) + .into_iter() + .filter_map(|(_, dep)| if dep.valid { dep.to } else { None }) + .collect(); + + for target_idx in resolved_edges { + let target_node = graph.get_node(target_idx).unwrap(); + // Only add workspace nodes to next level when from root + if target_node.is_workspace() && node_index == graph.root_index { + next_level.push(target_idx); + } + } + + // Collect only unresolved edges (these need name/spec cloned) + let unresolved_edges = collect_dependency_edges(graph, node_index); + + PROGRESS_BAR.inc_length(unresolved_edges.len() as u64); log_progress(&format!( "resolving {}", graph.get_node(node_index).unwrap().name )); - for edge_info in dependency_edges { - // Handle already resolved edges (e.g., workspace edges) - if edge_info.is_valid { - if let Some(target_idx) = edge_info.target { - let target_node = graph.get_node(target_idx).unwrap(); - // Only add workspace nodes to next level when from root - if target_node.is_workspace() && node_index == graph.root_index { - next_level.push(target_idx); - } - } - continue; - } - + for edge_info in unresolved_edges { tracing::debug!( "going to build deps {}@{} from [{:?}]", edge_info.name, @@ -131,14 +126,15 @@ pub async fn build_deps(graph: &mut DependencyGraph) -> Result<()> { ); // Check if there's an override for this dependency - let effective_spec = graph - .check_override(node_index, &edge_info.name, &edge_info.spec) - .unwrap_or_else(|| edge_info.spec.clone()); + let override_spec = + graph.check_override(node_index, &edge_info.name, &edge_info.spec); + let effective_spec: &str = + override_spec.as_deref().unwrap_or(&edge_info.spec); // Resolve dependency from registry with effective spec let resolved = match resolve_dependency( &edge_info.name, - &effective_spec, + effective_spec, &edge_info.edge_type, ) .await? @@ -173,12 +169,8 @@ pub async fn build_deps(graph: &mut DependencyGraph) -> Result<()> { ); // Create new node - let new_node = place_deps( - edge_info.name.clone(), - resolved.clone(), - conflict_parent, - graph, - ); + let new_node = + place_deps(&edge_info.name, resolved.clone(), conflict_parent, graph); let new_index = graph.add_node(new_node); // Add physical edge @@ -220,7 +212,7 @@ pub async fn build_deps(graph: &mut DependencyGraph) -> Result<()> { /// Create a new package node fn place_deps( - name: String, + name: &str, pkg: ResolvedPackage, parent: NodeIndex, graph: &DependencyGraph, @@ -234,7 +226,7 @@ fn place_deps( parent_node.path.join(format!("node_modules/{name}")) }; - let new_node = PackageNode::new(name.clone(), path, pkg.manifest); + let new_node = PackageNode::new(name.to_string(), path, pkg.manifest); tracing::debug!( "\nInstalling {}@{} under parent {:?}", @@ -275,12 +267,7 @@ async fn add_dependency_edges( ); for (name, version) in deps { let version_spec = version.as_str().unwrap_or("").to_string(); - graph.add_dependency_edge( - node_index, - name.clone(), - version_spec, - edge_type.clone(), - ); + graph.add_dependency_edge(node_index, name.clone(), version_spec, edge_type); tracing::debug!( "add edge {}@{} for {}", name, @@ -390,7 +377,7 @@ mod tests { manifest: json!({"name": "lodash", "version": "4.17.21"}), }; - let new_node = place_deps("lodash".to_string(), resolved, graph.root_index, &graph); + let new_node = place_deps("lodash", resolved, graph.root_index, &graph); assert_eq!(new_node.name, "lodash"); assert_eq!(new_node.version, "4.17.21"); diff --git a/crates/pm/src/helper/ruborist.rs b/crates/pm/src/helper/ruborist.rs index c889ed096..523c76847 100644 --- a/crates/pm/src/helper/ruborist.rs +++ b/crates/pm/src/helper/ruborist.rs @@ -77,7 +77,7 @@ impl Ruborist { graph.root_index, name.clone(), version_spec, - dep_type.clone(), + dep_type, ); tracing::debug!("add edge {}@{}", name, version); } @@ -151,7 +151,7 @@ impl Ruborist { workspace_index, dep_name.clone(), version_spec, - edge_type.clone(), + edge_type, ); tracing::debug!("add edge {}@{} for {}", dep_name, version, name); } diff --git a/crates/pm/src/model/node.rs b/crates/pm/src/model/node.rs index 7c1635b3e..94f4560d4 100644 --- a/crates/pm/src/model/node.rs +++ b/crates/pm/src/model/node.rs @@ -1,5 +1,5 @@ /// Edge type in dependency graph -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] pub enum EdgeType { Prod, // Production dependency Dev, // Development dependency From f445ec317f949fa86fb60460c4d8a7be4a1c67c4 Mon Sep 17 00:00:00 2001 From: elrrrrrrr Date: Tue, 21 Oct 2025 21:41:58 +0800 Subject: [PATCH 3/3] chore: manifest clone --- crates/pm/src/helper/graph_builder.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/crates/pm/src/helper/graph_builder.rs b/crates/pm/src/helper/graph_builder.rs index 4becf23af..9c0f82bf1 100644 --- a/crates/pm/src/helper/graph_builder.rs +++ b/crates/pm/src/helper/graph_builder.rs @@ -170,7 +170,7 @@ pub async fn build_deps(graph: &mut DependencyGraph) -> Result<()> { // Create new node let new_node = - place_deps(&edge_info.name, resolved.clone(), conflict_parent, graph); + place_deps(&edge_info.name, &resolved, conflict_parent, graph); let new_index = graph.add_node(new_node); // Add physical edge @@ -213,7 +213,7 @@ pub async fn build_deps(graph: &mut DependencyGraph) -> Result<()> { /// Create a new package node fn place_deps( name: &str, - pkg: ResolvedPackage, + pkg: &ResolvedPackage, parent: NodeIndex, graph: &DependencyGraph, ) -> PackageNode { @@ -226,7 +226,7 @@ fn place_deps( parent_node.path.join(format!("node_modules/{name}")) }; - let new_node = PackageNode::new(name.to_string(), path, pkg.manifest); + let new_node = PackageNode::new(name.to_string(), path, pkg.manifest.clone()); tracing::debug!( "\nInstalling {}@{} under parent {:?}", @@ -377,7 +377,7 @@ mod tests { manifest: json!({"name": "lodash", "version": "4.17.21"}), }; - let new_node = place_deps("lodash", resolved, graph.root_index, &graph); + let new_node = place_deps("lodash", &resolved, graph.root_index, &graph); assert_eq!(new_node.name, "lodash"); assert_eq!(new_node.version, "4.17.21");