diff --git a/Cargo.lock b/Cargo.lock index c138b34..67461e5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -424,6 +424,7 @@ dependencies = [ "ra_ap_project_model", "ra_ap_syntax", "ra_ap_vfs", + "regex", "semver", "serde", "serde_json", @@ -3828,9 +3829,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.10.5" +version = "1.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b91213439dad192326a0d7c6ee3955910425f441d7038e0d6933b0aec5c4517f" +checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" dependencies = [ "aho-corasick", "memchr", diff --git a/Cargo.toml b/Cargo.toml index 112f961..4be8aa0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,6 +52,7 @@ ra_ap_project_model = "0.0.185" ra_ap_syntax = "0.0.185" ra_ap_vfs = "0.0.185" ra_ap_cfg = "0.0.185" +regex = "1.10.6" serde = { workspace = true, features = ["derive"] } serde_json.workspace = true serde_with.workspace = true @@ -73,4 +74,4 @@ serde = { version = "1.0.192", features = ["derive"] } anyhow = "1.0.75" env_logger = "0.10.1" home = "0.5.5" -serde_with = "3.4.0" \ No newline at end of file +serde_with = "3.4.0" diff --git a/Makefile b/Makefile index 56eb1cc..afcde29 100644 --- a/Makefile +++ b/Makefile @@ -4,10 +4,7 @@ SCAN_ALL := cargo run --release --bin scan_all -- UPDATE_TEST_CRATES_CSV := ./scripts/update_test_crates_csv.py -dependencies: - - cargo install cargo-download - -install: dependencies +install: cargo build && cargo build --release checks: diff --git a/README.md b/README.md index 69a9e57..72fdb91 100644 --- a/README.md +++ b/README.md @@ -7,11 +7,8 @@ Cargo Scan is a tool for auditing Rust crates. ## Installation 1. Clone this repository. -2. Make sure you have [Rust](https://www.rust-lang.org/tools/install). -3. Run `rustup update` to ensure you have the latest version of Rust. -4. Run `make install`. - -This installs [cargo-download](https://crates.io/crates/cargo-download) and builds the Rust source. +2. Run `rustup update` to ensure you have the latest version of Rust (or install it via the [official website]((https://www.rust-lang.org/tools/install))). +3. Run `cargo build`. Known working builds: diff --git a/src/bin/scan_all.rs b/src/bin/scan_all.rs index 111d98f..5ec97fe 100644 --- a/src/bin/scan_all.rs +++ b/src/bin/scan_all.rs @@ -5,6 +5,7 @@ //! //! See README for current usage information. +use cargo_scan::download_crate; use cargo_scan::effect::EffectInstance; use cargo_scan::scan_stats::{self, CrateStats}; use cargo_scan::util; @@ -15,7 +16,6 @@ use std::collections::HashMap; use std::fs; use std::io::Write; use std::path::{Path, PathBuf}; -use std::process::Command; use std::sync::mpsc; use threadpool::ThreadPool; @@ -38,7 +38,7 @@ const RESULTS_PATTERNS_SUFFIX: &str = "_patterns.csv"; const RESULTS_METADATA_SUFFIX: &str = "_metadata.csv"; // Whether to remove and re-download old downloaded packages -const UPDATE_DOWNLOADS: bool = false; +const UPDATE_DOWNLOADS: bool = true; /* CLI @@ -83,21 +83,15 @@ fn crate_stats( let output_dir = download_loc.join(Path::new(crt)); if !test_run { - if UPDATE_DOWNLOADS { + if UPDATE_DOWNLOADS && output_dir.is_dir() { fs::remove_dir_all(&output_dir).expect("failed to remove old dir"); } if !output_dir.is_dir() { - info!("Downloading {} to: {:?}", crt, output_dir); - - let _output = Command::new("cargo") - .arg("download") - .arg("-x") - .arg(crt) - .arg("-o") - .arg(&output_dir) - .output() - .expect("failed to run cargo download"); + info!("Downloading {} to: {}", crt, output_dir.to_string_lossy()); + + download_crate::download_latest_crate_version(crt, CRATES_DIR) + .expect("failed to download crate"); } } diff --git a/src/download_crate.rs b/src/download_crate.rs index 3502202..c8f6b37 100644 --- a/src/download_crate.rs +++ b/src/download_crate.rs @@ -1,13 +1,19 @@ use std::fs::{create_dir_all, remove_file, write, File}; use std::path::PathBuf; +use std::process::Command; -use anyhow::Result; +use anyhow::{anyhow, Result}; use cargo_lock::Package; use curl::easy::Easy; use flate2::read::GzDecoder; use log::info; +use regex::Regex; use tar::Archive; +// Regexes to match crate names and versions +const CRATE_NAME_REGEX: &str = r"[a-zA-Z0-9_-]+"; +const SEMVER_REGEX: &str = r"(0|[1-9]\d*)\.(0|[1-9]\d*)\.(0|[1-9]\d*)(?:-((?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+([0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?"; + fn get_crates_io_url(package_name: &str, package_version: &str) -> String { format!( "https://crates.io/api/v1/crates/{}/{}/download", @@ -86,6 +92,54 @@ pub fn download_crate_from_info( download_crate(&url, package_name, package_version, download_dir) } +/// Get the latest version of a crate from only the package name. +pub fn get_latest_version(package_name: &str) -> Result { + // Query `cargo search`. + let result = Command::new("cargo") + .arg("search") + .arg(package_name) + .arg("--limit") + .arg("1") + .output()?; + + // Convert the output to a string. + let output = String::from_utf8(result.stdout)?; + + // Parse the output. It should contain = "" + let re_str = format!("^({}) = \"({})\"", CRATE_NAME_REGEX, SEMVER_REGEX); + let re = Regex::new(&re_str).unwrap(); + if let Some(caps) = re.captures(&output) { + // Debug + // println!("Match found: {:?}", caps); + + let name = &caps[1]; + let version = &caps[2]; + if name == package_name { + // Debug + // println!("Found version: {} for package: {}", version, package_name); + + return Ok(version.to_string()); + } + } + + Err(anyhow!("No match found for package name: {}", package_name)) +} + +/// Downloads the latest version of a crate from only the package name +/// Also, moves the output directory to just use the package name in the final output folder. +pub fn download_latest_crate_version( + package_name: &str, + download_dir: &str, +) -> Result { + let latest_version = get_latest_version(package_name)?; + let result = download_crate_from_info(package_name, &latest_version, download_dir)?; + let _output = Command::new("mv") + .arg(format!("{}/{}-{}", download_dir, package_name, latest_version)) + .arg(format!("{}/{}", download_dir, package_name)) + .output()?; + Ok(result) +} + /// Downloads the crate from the `cargo_lock::Package` pub fn download_crate_from_package( package: &Package,