From 90b214ee460c9bfc32e12d3632e63cf6b76b4d4a Mon Sep 17 00:00:00 2001 From: Paolo Barbolini Date: Sat, 6 Apr 2024 16:38:53 +0200 Subject: [PATCH] feat: parallelize implementation --- Cargo.lock | 52 +++++++++++++++++++++++++++++++++++++++ Cargo.toml | 1 + src/git.rs | 2 +- src/main.rs | 71 +++++++++++++++++++++++++++++++++-------------------- 4 files changed, 99 insertions(+), 27 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index dbbe979..876fe76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -91,6 +91,7 @@ dependencies = [ "cargo-lock", "cargo_toml", "flate2", + "rayon", "reqwest", "semver", "serde", @@ -168,6 +169,31 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "crossbeam-deque" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "613f8cc01fe9cf1a3eb3d7f488fd2fa8388403e97039e2f73692932e291a770d" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" + [[package]] name = "crypto-common" version = "0.1.6" @@ -188,6 +214,12 @@ dependencies = [ "crypto-common", ] +[[package]] +name = "either" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" + [[package]] name = "equivalent" version = "1.0.1" @@ -604,6 +636,26 @@ dependencies = [ "proc-macro2", ] +[[package]] +name = "rayon" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.4.1" diff --git a/Cargo.toml b/Cargo.toml index 7c760a4..479a0a2 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,3 +20,4 @@ semver = "1" serde = { version = "1", features = ["derive"] } serde_json = "1" anyhow = "1.0.14" +rayon = "1" diff --git a/src/git.rs b/src/git.rs index 0de3d28..c3b7334 100644 --- a/src/git.rs +++ b/src/git.rs @@ -16,7 +16,7 @@ use crate::package::Package; #[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] pub struct GitUrl(Url); -#[derive(Debug)] +#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)] pub struct GitRepository { repo_dir: PathBuf, } diff --git a/src/main.rs b/src/main.rs index d9bfec1..f215910 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,4 +1,5 @@ use std::{ + collections::BTreeMap, env, fs, io::{self as std_io, Read}, path::Path, @@ -9,6 +10,7 @@ use anyhow::{ensure, Context, Result}; use cargo_lock::{package::SourceKind, Checksum, Lockfile}; use cargo_toml::Manifest; use git::GitUrl; +use rayon::iter::{IntoParallelIterator, ParallelIterator}; use serde::Deserialize; use sha2::{Digest as _, Sha256}; use url::Url; @@ -76,45 +78,62 @@ fn main() -> Result<()> { let lock = Lockfile::load(lock).context("decode Cargo.lock")?; - for lock_info in lock.packages { - let name = lock_info.name.clone(); - let version = lock_info.version.clone(); + let resolved_packages = lock + .packages + .into_par_iter() + .filter_map(|lock_info| { + let name = lock_info.name.clone(); + let version = lock_info.version.clone(); - let resolved_package = match resolve_package(&http_client, &crates_dir, lock_info) { - Ok(resolved_package) => resolved_package, - Err(err) => { - println!("Couldn't resolve package {name} v{version} err={err:?}"); - continue; + match resolve_package(&http_client, &crates_dir, lock_info) { + Ok(resolved_package) => Some(resolved_package), + Err(err) => { + println!("Couldn't resolve package {name} v{version} err={err:?}"); + None + } } - }; + }) + .collect::>(); + + let mut grouped_resolved_packages = BTreeMap::<_, Vec<_>>::new(); + for resolved_package in resolved_packages { + grouped_resolved_packages + .entry(resolved_package.repository_url.clone()) + .or_default() + .push(resolved_package); + } - let mut git_repository = - match GitRepository::obtain(&repos_dir, resolved_package.repository_url.clone()) { + grouped_resolved_packages + .into_par_iter() + .for_each(|(repository_url, resolved_packages)| { + let mut git_repository = match GitRepository::obtain(&repos_dir, repository_url) { Ok(git_repository) => git_repository, Err(err) => { println!( "Couldn't obtain git repository for {} v{} err={:?} url={}", + resolved_packages[0].lock_info.name, + resolved_packages[0].lock_info.version, + err, + resolved_packages[0].repository_url + ); + return; + } + }; + + for resolved_package in resolved_packages { + if let Err(err) = + analyze_package(&default_toolchain, &resolved_package, &mut git_repository) + { + println!( + "Couldn't analyze package for {} v{} err={:?} url={}", resolved_package.lock_info.name, resolved_package.lock_info.version, err, resolved_package.repository_url ); - continue; } - }; - - if let Err(err) = - analyze_package(&default_toolchain, &resolved_package, &mut git_repository) - { - println!( - "Couldn't analyze package for {} v{} err={:?} url={}", - resolved_package.lock_info.name, - resolved_package.lock_info.version, - err, - resolved_package.repository_url - ); - } - } + } + }); Ok(()) }