From d53d74e294be823af014a5cf7739992fa38ca520 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Mon, 2 May 2022 19:46:14 +0200 Subject: [PATCH 01/52] Extensions subcommand entry points --- cli/Cargo.toml | 3 +- cli/src/app.rs | 5 ++ cli/src/bin/phylum.rs | 5 ++ cli/src/commands/extensions/mod.rs | 90 ++++++++++++++++++++++++++++++ cli/src/commands/mod.rs | 2 + cli/src/config.rs | 15 ++++- 6 files changed, 117 insertions(+), 3 deletions(-) create mode 100644 cli/src/commands/extensions/mod.rs diff --git a/cli/Cargo.toml b/cli/Cargo.toml index a62c328b8..3068c2236 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -5,9 +5,10 @@ authors = ["Eric Freitag "] edition = "2018" [features] -default = ["selfmanage"] +default = ["selfmanage", "extensions"] phylum-online = [] selfmanage = [] +extensions = [] # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/cli/src/app.rs b/cli/src/app.rs index e88a7e6a1..71b6e24f3 100644 --- a/cli/src/app.rs +++ b/cli/src/app.rs @@ -138,6 +138,11 @@ pub fn app<'a>() -> clap::Command<'a> { .about("Display application version") ); + #[cfg(feature = "extensions")] + { + app = app.subcommand(crate::commands::extensions::command()); + } + #[cfg(feature = "selfmanage")] { app = app.subcommand( diff --git a/cli/src/bin/phylum.rs b/cli/src/bin/phylum.rs index 7d406c9c7..aa78617f6 100644 --- a/cli/src/bin/phylum.rs +++ b/cli/src/bin/phylum.rs @@ -10,6 +10,8 @@ use spinners::{Spinner, Spinners}; use phylum_cli::api::PhylumApi; use phylum_cli::commands::auth::*; +#[cfg(feature = "extensions")] +use phylum_cli::commands::extensions::*; use phylum_cli::commands::jobs::*; use phylum_cli::commands::packages::*; use phylum_cli::commands::project::handle_project; @@ -204,6 +206,9 @@ async fn handle_commands() -> CommandResult { return handle_submission(&mut api, config, &matches).await; } else if let Some(matches) = matches.subcommand_matches("history") { return handle_history(&mut api, matches).await; + } else if let Some(matches) = matches.subcommand_matches("extensions") { + #[cfg(feature = "extensions")] + return handle_extensions(matches).await; } else if should_cancel { if let Some(matches) = matches.subcommand_matches("cancel") { let request_id = matches.value_of("request_id").unwrap().to_string(); diff --git a/cli/src/commands/extensions/mod.rs b/cli/src/commands/extensions/mod.rs new file mode 100644 index 000000000..c940ee953 --- /dev/null +++ b/cli/src/commands/extensions/mod.rs @@ -0,0 +1,90 @@ +use std::{convert::TryFrom, path::PathBuf}; + +use crate::commands::{CommandResult, CommandValue, ExitCode}; + +use anyhow::{anyhow, Result}; +use clap::{arg, ArgMatches, Command, ValueHint}; + +pub struct Extension { + path: PathBuf, + name: String, +} + +// Load the extension from the specified path. +impl TryFrom for Extension { + type Error = anyhow::Error; + + fn try_from(path: PathBuf) -> Result { + todo!() + } +} + +pub fn command<'a>() -> Command<'a> { + Command::new("extension") + .about("Run extensions") + .subcommand( + Command::new("add") + .about("Install extension") + .arg(arg!([PATH]).required(true).value_hint(ValueHint::FilePath)), + ) + .subcommand( + Command::new("remove") + .about("Uninstall extension") + .arg(arg!([NAME]).required(true)), + ) + .subcommand(Command::new("list").about("List installed extensions")) +} + +/// Generate the subcommands for each extension. +pub fn extensions_subcommands<'a>(command: Command<'a>) -> Result> { + let extensions = list_extensions()? + .into_iter() + .filter(|ext| command.get_subcommands().all(|p| p.get_name() != ext.name)) + .collect::>(); + + Ok(extensions.into_iter().fold(command, |command, ext| { + command.subcommand(Command::new(&ext.name)) + })) +} + +/// Entry point for the `extensions` subcommand. +pub async fn handle_extensions(matches: &ArgMatches) -> CommandResult { + if let Some(matches) = matches.subcommand_matches("add") { + subcmd_add_extension(matches.value_of_t("PATH")?).await + } else if let Some(matches) = matches.subcommand_matches("remove") { + subcmd_remove_extension(matches.value_of("NAME").unwrap()).await + } else { + // also covers the `list` case + subcmd_list_extensions().await + } +} + +/// Handle the `extension add` subcommand path. +/// Add the extension from the specified path. +pub async fn subcmd_add_extension(path: PathBuf) -> CommandResult { + Ok(CommandValue::Code(ExitCode::Ok)) +} + +/// Handle the `extension remove` subcommand path. +/// Remove the extension named as specified. +pub async fn subcmd_remove_extension(name: &str) -> CommandResult { + Ok(CommandValue::Code(ExitCode::Ok)) +} + +/// Handle the `extension` / `extension list` subcommand path. +/// List installed extensions. +pub async fn subcmd_list_extensions() -> CommandResult { + Ok(CommandValue::Code(ExitCode::Ok)) +} + +// Return a list of installed extensions. Filter out invalid extensions instead of exiting early. +fn list_extensions() -> Result> { + Ok(std::fs::read_dir(extensions_path()?)? + .filter_map(|d| Extension::try_from(d.map(|d| d.path()).ok()?).ok()) + .collect::>()) +} + +// Construct and return the extension path. +fn extensions_path() -> Result { + Ok(crate::config::data_dir()?.join("phylum").join("extensions")) +} diff --git a/cli/src/commands/mod.rs b/cli/src/commands/mod.rs index 3a5ac8328..158784f02 100644 --- a/cli/src/commands/mod.rs +++ b/cli/src/commands/mod.rs @@ -5,6 +5,8 @@ pub mod jobs; pub mod lock_files; pub mod packages; pub mod project; +#[cfg(feature = "extensions")] +pub mod extensions; #[cfg(feature = "selfmanage")] pub mod uninstall; diff --git a/cli/src/config.rs b/cli/src/config.rs index 044a0e22e..567a31a1c 100644 --- a/cli/src/config.rs +++ b/cli/src/config.rs @@ -169,8 +169,7 @@ pub fn get_home_settings_path() -> Result { let home_path = home::home_dir().ok_or_else(|| anyhow!("Couldn't find the user's home directory"))?; - let config_dir = config_dir(&home_path); - let config_path = config_dir.join("phylum").join("settings.yaml"); + let config_path = config_dir(&home_path).join("phylum").join("settings.yaml"); let old_config_path = home_path.join(".phylum").join("settings.yaml"); // Migrate the config from the old location. @@ -195,6 +194,18 @@ pub fn get_home_settings_path() -> Result { Ok(config_path) } +/// Resolve XDG data directory. +pub fn data_dir() -> Result { + let home_path = + home::home_dir().ok_or_else(|| anyhow!("Couldn't find the user's home directory"))?; + + Ok(env::var("XDG_DATA_HOME") + .ok() + .filter(|s| !s.is_empty()) + .map(PathBuf::from) + .unwrap_or_else(|| home_path.join(".local").join("share"))) +} + /// Resolve XDG config directory. pub fn config_dir(home_path: &Path) -> PathBuf { env::var("XDG_CONFIG_HOME") From 29e0323e543532578dd30fa8dde168ca827f2214 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Tue, 3 May 2022 19:17:28 +0200 Subject: [PATCH 02/52] Construct extension from path (with manifest) --- cli/src/commands/extensions/mod.rs | 80 +++++++++++++++++-- .../sample-extension/PhylumExt.toml | 3 + .../sample-extension/sample_extension.sh | 3 + 3 files changed, 81 insertions(+), 5 deletions(-) create mode 100644 cli/tests/fixtures/extensions/sample-extension/PhylumExt.toml create mode 100644 cli/tests/fixtures/extensions/sample-extension/sample_extension.sh diff --git a/cli/src/commands/extensions/mod.rs b/cli/src/commands/extensions/mod.rs index c940ee953..dc64bae85 100644 --- a/cli/src/commands/extensions/mod.rs +++ b/cli/src/commands/extensions/mod.rs @@ -1,13 +1,24 @@ -use std::{convert::TryFrom, path::PathBuf}; +use std::{convert::TryFrom, fs::File, io::Read, path::PathBuf}; use crate::commands::{CommandResult, CommandValue, ExitCode}; use anyhow::{anyhow, Result}; use clap::{arg, ArgMatches, Command, ValueHint}; +use serde::Deserialize; +const MANIFEST_NAME: &str = "PhylumExt.toml"; + +#[derive(Debug)] pub struct Extension { path: PathBuf, + manifest: ExtensionManifest, +} + +#[derive(Deserialize, Debug)] +pub struct ExtensionManifest { name: String, + description: Option, + entry_point: String, } // Load the extension from the specified path. @@ -15,7 +26,44 @@ impl TryFrom for Extension { type Error = anyhow::Error; fn try_from(path: PathBuf) -> Result { - todo!() + if !path.is_dir() { + return Err(anyhow!("{}: not a directory", path.to_string_lossy())); + } + + let manifest_path = path.join(MANIFEST_NAME); + if !manifest_path.exists() { + return Err(anyhow!( + "{}: missing {}", + path.to_string_lossy(), + MANIFEST_NAME + )); + } + + let mut buf = Vec::new(); + File::open(manifest_path)?.read_to_end(&mut buf)?; + + let manifest: ExtensionManifest = toml::from_slice(&buf)?; + let entry_point_path = path.join(&manifest.entry_point); + + if !entry_point_path.exists() { + return Err(anyhow!( + "{}: entry point does not exist", + entry_point_path.to_string_lossy() + )); + } + + if !entry_point_path.is_file() { + return Err(anyhow!( + "{}: entry point is not a file", + entry_point_path.to_string_lossy() + )); + } + + // TODO add further validation if necessary: + // - Check that the name matches /^[a-z0-9-_]+$/ + // - Check that the entry point is a supported format (.wasm?) + // - Check that the entry point is appropriately signed + Ok(Extension { path, manifest }) } } @@ -39,11 +87,15 @@ pub fn command<'a>() -> Command<'a> { pub fn extensions_subcommands<'a>(command: Command<'a>) -> Result> { let extensions = list_extensions()? .into_iter() - .filter(|ext| command.get_subcommands().all(|p| p.get_name() != ext.name)) + .filter(|ext| { + command + .get_subcommands() + .all(|p| p.get_name() != ext.manifest.name) + }) .collect::>(); Ok(extensions.into_iter().fold(command, |command, ext| { - command.subcommand(Command::new(&ext.name)) + command.subcommand(Command::new(&ext.manifest.name)) })) } @@ -62,18 +114,36 @@ pub async fn handle_extensions(matches: &ArgMatches) -> CommandResult { /// Handle the `extension add` subcommand path. /// Add the extension from the specified path. pub async fn subcmd_add_extension(path: PathBuf) -> CommandResult { + let extension = Extension::try_from(path)?; + Ok(CommandValue::Code(ExitCode::Ok)) } /// Handle the `extension remove` subcommand path. /// Remove the extension named as specified. pub async fn subcmd_remove_extension(name: &str) -> CommandResult { + let extension = Extension::try_from(extensions_path()?.join(name))?; + Ok(CommandValue::Code(ExitCode::Ok)) } /// Handle the `extension` / `extension list` subcommand path. /// List installed extensions. pub async fn subcmd_list_extensions() -> CommandResult { + let extensions = list_extensions()?; + + extensions.into_iter().for_each(|ext| { + println!( + "{:20} {}", + ext.manifest.name, + ext.manifest + .description + .as_ref() + .map(String::as_str) + .unwrap_or("") + ); + }); + Ok(CommandValue::Code(ExitCode::Ok)) } @@ -84,7 +154,7 @@ fn list_extensions() -> Result> { .collect::>()) } -// Construct and return the extension path. +// Construct and return the extension path: $XDG_DATA_HOME/phylum/extensions fn extensions_path() -> Result { Ok(crate::config::data_dir()?.join("phylum").join("extensions")) } diff --git a/cli/tests/fixtures/extensions/sample-extension/PhylumExt.toml b/cli/tests/fixtures/extensions/sample-extension/PhylumExt.toml new file mode 100644 index 000000000..ac0eb6d20 --- /dev/null +++ b/cli/tests/fixtures/extensions/sample-extension/PhylumExt.toml @@ -0,0 +1,3 @@ +name = "Sample extension" +description = "This extension does a thing" +entry_point = "sample_extension.sh" diff --git a/cli/tests/fixtures/extensions/sample-extension/sample_extension.sh b/cli/tests/fixtures/extensions/sample-extension/sample_extension.sh new file mode 100644 index 000000000..facae01d1 --- /dev/null +++ b/cli/tests/fixtures/extensions/sample-extension/sample_extension.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +echo "Hello, world!" From d7d53e4f3730678cf347dd90d9704dff1d0670fc Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Wed, 4 May 2022 18:21:37 +0200 Subject: [PATCH 03/52] Install and load methods for Extension --- Cargo.lock | 21 ++++ cli/Cargo.toml | 1 + cli/src/commands/extensions/extension.rs | 112 ++++++++++++++++++ cli/src/commands/extensions/mod.rs | 88 ++------------ cli/tests/extensions.rs | 71 +++++++++++ .../sample-extension/PhylumExt.toml | 2 +- 6 files changed, 214 insertions(+), 81 deletions(-) create mode 100644 cli/src/commands/extensions/extension.rs create mode 100644 cli/tests/extensions.rs diff --git a/Cargo.lock b/Cargo.lock index 0bce7cf1e..f0b391ccd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1257,6 +1257,7 @@ dependencies = [ "unicode-width", "url", "uuid", + "walkdir", "wiremock", "yaml-rust", "zeroize", @@ -1621,6 +1622,15 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "73b4b750c782965c211b42f022f59af1fbceabdd026623714f104152f1ec149f" +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "scopeguard" version = "1.1.0" @@ -2181,6 +2191,17 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" +[[package]] +name = "walkdir" +version = "2.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "808cf2735cd4b6866113f648b791c6adc5714537bc222d9347bb203386ffda56" +dependencies = [ + "same-file", + "winapi", + "winapi-util", +] + [[package]] name = "want" version = "0.3.0" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 3068c2236..f4b8ad77a 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -57,6 +57,7 @@ yaml-rust = "*" zeroize = "1.4.0" zip = "0.6.2" unicode-width = "0.1.9" +walkdir = "2.3.2" [dev-dependencies] assert_cmd = "2.0.4" diff --git a/cli/src/commands/extensions/extension.rs b/cli/src/commands/extensions/extension.rs new file mode 100644 index 000000000..77d5a7d6f --- /dev/null +++ b/cli/src/commands/extensions/extension.rs @@ -0,0 +1,112 @@ +use std::{convert::TryFrom, fs::File, io::Read, path::PathBuf}; + +use anyhow::{anyhow, Result}; +use serde::Deserialize; + +const MANIFEST_NAME: &str = "PhylumExt.toml"; + +#[derive(Debug)] +pub struct Extension { + path: PathBuf, + manifest: ExtensionManifest, +} + +#[derive(Deserialize, Debug)] +pub struct ExtensionManifest { + name: String, + description: Option, + entry_point: String, +} + +impl Extension { + pub fn name(&self) -> &str { + &self.manifest.name + } + + pub fn description(&self) -> Option<&str> { + self.manifest.description.as_ref().map(String::as_str) + } + + /// Install the extension in the default path. + pub fn install(&self) -> Result<()> { + let target_prefix = extensions_path()?.join(self.name()); + + if target_prefix == self.path { + return Err(anyhow!("extension path and installation path are identical, skipping")); + } + + for entry in walkdir::WalkDir::new(&self.path) { + let source_path = entry?.into_path(); + let dest_path = target_prefix.join(source_path.strip_prefix(&self.path)?); + + if source_path.is_dir() { + std::fs::create_dir_all(dest_path)?; + } else if source_path.is_file() { + if dest_path.exists() { + return Err(anyhow!("{}: already exists", dest_path.to_string_lossy())); + } else { + std::fs::copy(source_path, dest_path)?; + } + } + } + + Ok(()) + } + + /// Load an extension from the default path. + pub fn load(name: &str) -> Result { + Extension::try_from(extensions_path()?.join(name)) + } +} + +// Load the extension from the specified path. +impl TryFrom for Extension { + type Error = anyhow::Error; + + fn try_from(path: PathBuf) -> Result { + if !path.is_dir() { + return Err(anyhow!("{}: not a directory", path.to_string_lossy())); + } + + let manifest_path = path.join(MANIFEST_NAME); + if !manifest_path.exists() { + return Err(anyhow!( + "{}: missing {}", + path.to_string_lossy(), + MANIFEST_NAME + )); + } + + let mut buf = Vec::new(); + File::open(manifest_path)?.read_to_end(&mut buf)?; + + let manifest: ExtensionManifest = toml::from_slice(&buf)?; + let entry_point_path = path.join(&manifest.entry_point); + + if !entry_point_path.exists() { + return Err(anyhow!( + "{}: entry point does not exist", + entry_point_path.to_string_lossy() + )); + } + + if !entry_point_path.is_file() { + return Err(anyhow!( + "{}: entry point is not a file", + entry_point_path.to_string_lossy() + )); + } + + // TODO add further validation if necessary: + // - Check that the name matches /^[a-z0-9-_]+$/ + // - Check that the entry point is a supported format (.wasm?) + // - Check that the entry point is appropriately signed + Ok(Extension { path, manifest }) + } +} + + +// Construct and return the extension path: $XDG_DATA_HOME/phylum/extensions +pub fn extensions_path() -> Result { + Ok(crate::config::data_dir()?.join("phylum").join("extensions")) +} diff --git a/cli/src/commands/extensions/mod.rs b/cli/src/commands/extensions/mod.rs index dc64bae85..f8177f80a 100644 --- a/cli/src/commands/extensions/mod.rs +++ b/cli/src/commands/extensions/mod.rs @@ -1,71 +1,12 @@ -use std::{convert::TryFrom, fs::File, io::Read, path::PathBuf}; +pub mod extension; +use std::{convert::TryFrom, path::PathBuf}; + +pub use extension::*; use crate::commands::{CommandResult, CommandValue, ExitCode}; -use anyhow::{anyhow, Result}; +use anyhow::Result; use clap::{arg, ArgMatches, Command, ValueHint}; -use serde::Deserialize; - -const MANIFEST_NAME: &str = "PhylumExt.toml"; - -#[derive(Debug)] -pub struct Extension { - path: PathBuf, - manifest: ExtensionManifest, -} - -#[derive(Deserialize, Debug)] -pub struct ExtensionManifest { - name: String, - description: Option, - entry_point: String, -} - -// Load the extension from the specified path. -impl TryFrom for Extension { - type Error = anyhow::Error; - - fn try_from(path: PathBuf) -> Result { - if !path.is_dir() { - return Err(anyhow!("{}: not a directory", path.to_string_lossy())); - } - - let manifest_path = path.join(MANIFEST_NAME); - if !manifest_path.exists() { - return Err(anyhow!( - "{}: missing {}", - path.to_string_lossy(), - MANIFEST_NAME - )); - } - - let mut buf = Vec::new(); - File::open(manifest_path)?.read_to_end(&mut buf)?; - - let manifest: ExtensionManifest = toml::from_slice(&buf)?; - let entry_point_path = path.join(&manifest.entry_point); - - if !entry_point_path.exists() { - return Err(anyhow!( - "{}: entry point does not exist", - entry_point_path.to_string_lossy() - )); - } - - if !entry_point_path.is_file() { - return Err(anyhow!( - "{}: entry point is not a file", - entry_point_path.to_string_lossy() - )); - } - - // TODO add further validation if necessary: - // - Check that the name matches /^[a-z0-9-_]+$/ - // - Check that the entry point is a supported format (.wasm?) - // - Check that the entry point is appropriately signed - Ok(Extension { path, manifest }) - } -} pub fn command<'a>() -> Command<'a> { Command::new("extension") @@ -90,12 +31,12 @@ pub fn extensions_subcommands<'a>(command: Command<'a>) -> Result> { .filter(|ext| { command .get_subcommands() - .all(|p| p.get_name() != ext.manifest.name) + .all(|p| p.get_name() != ext.name()) }) .collect::>(); Ok(extensions.into_iter().fold(command, |command, ext| { - command.subcommand(Command::new(&ext.manifest.name)) + command.subcommand(Command::new(ext.name())) })) } @@ -133,15 +74,7 @@ pub async fn subcmd_list_extensions() -> CommandResult { let extensions = list_extensions()?; extensions.into_iter().for_each(|ext| { - println!( - "{:20} {}", - ext.manifest.name, - ext.manifest - .description - .as_ref() - .map(String::as_str) - .unwrap_or("") - ); + println!("{:20} {}", ext.name(), ext.description().unwrap_or("")); }); Ok(CommandValue::Code(ExitCode::Ok)) @@ -153,8 +86,3 @@ fn list_extensions() -> Result> { .filter_map(|d| Extension::try_from(d.map(|d| d.path()).ok()?).ok()) .collect::>()) } - -// Construct and return the extension path: $XDG_DATA_HOME/phylum/extensions -fn extensions_path() -> Result { - Ok(crate::config::data_dir()?.join("phylum").join("extensions")) -} diff --git a/cli/tests/extensions.rs b/cli/tests/extensions.rs new file mode 100644 index 000000000..c059a02ee --- /dev/null +++ b/cli/tests/extensions.rs @@ -0,0 +1,71 @@ +use std::convert::TryFrom; +use std::path::{Path, PathBuf}; + +use phylum_cli::commands::extensions::*; +use rand::Rng; + +#[test] +fn valid_extension_is_loaded_correctly() { + let ext = Extension::try_from(fixtures_path().join("sample-extension")).unwrap(); + + assert_eq!(ext.name(), "sample-extension"); +} + +#[test] +fn extension_is_installed_correctly() { + let tmp_dir = TmpDir::new(); + std::env::set_var("XDG_DATA_HOME", tmp_dir.0.as_os_str()); + + let ext = Extension::try_from(fixtures_path().join("sample-extension")).unwrap(); + ext.install().unwrap(); + + let installed_ext = Extension::load(ext.name()).unwrap(); +} + +fn project_root() -> PathBuf { + Path::new(&env!("CARGO_MANIFEST_DIR")) + .ancestors() + .nth(1) + .unwrap() + .to_path_buf() +} + +fn fixtures_path() -> PathBuf { + project_root() + .join("cli") + .join("tests") + .join("fixtures") + .join("extensions") +} + +struct TmpDir(PathBuf); + +impl TmpDir { + fn new() -> Self { + let dir_name: String = rand::thread_rng() + .sample_iter(&rand::distributions::Alphanumeric) + .take(16) + .map(char::from) + .collect(); + + let path = tmp_path().join(dir_name); + std::fs::create_dir_all(&path).unwrap(); + Self(path) + } +} + +impl AsRef for TmpDir { + fn as_ref(&self) -> &Path { + &self.0 + } +} + +impl Drop for TmpDir { + fn drop(&mut self) { + std::fs::remove_dir_all(&self.0).unwrap(); + } +} + +fn tmp_path() -> PathBuf { + project_root().join("target").join("tests-tmp") +} diff --git a/cli/tests/fixtures/extensions/sample-extension/PhylumExt.toml b/cli/tests/fixtures/extensions/sample-extension/PhylumExt.toml index ac0eb6d20..6ec4ad0cc 100644 --- a/cli/tests/fixtures/extensions/sample-extension/PhylumExt.toml +++ b/cli/tests/fixtures/extensions/sample-extension/PhylumExt.toml @@ -1,3 +1,3 @@ -name = "Sample extension" +name = "sample-extension" description = "This extension does a thing" entry_point = "sample_extension.sh" From 596f7e730f07090134cd4e520b79f4dfbfc2a424 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Thu, 5 May 2022 17:27:00 +0200 Subject: [PATCH 04/52] Acceptance criteria tests --- Cargo.lock | 1 + cli/Cargo.toml | 1 + cli/src/bin/phylum.rs | 6 +- cli/src/commands/extensions/extension.rs | 19 +++ cli/src/commands/extensions/mod.rs | 9 +- cli/tests/extensions.rs | 156 +++++++++++++++++++++-- 6 files changed, 180 insertions(+), 12 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index f0b391ccd..3f43d7b67 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1239,6 +1239,7 @@ dependencies = [ "phylum_types", "prettytable-rs", "rand 0.8.5", + "regex", "reqwest", "routerify", "serde", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index f4b8ad77a..223b09bc5 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -58,6 +58,7 @@ zeroize = "1.4.0" zip = "0.6.2" unicode-width = "0.1.9" walkdir = "2.3.2" +regex = "1.5.5" [dev-dependencies] assert_cmd = "2.0.4" diff --git a/cli/src/bin/phylum.rs b/cli/src/bin/phylum.rs index aa78617f6..aa3257e34 100644 --- a/cli/src/bin/phylum.rs +++ b/cli/src/bin/phylum.rs @@ -206,9 +206,6 @@ async fn handle_commands() -> CommandResult { return handle_submission(&mut api, config, &matches).await; } else if let Some(matches) = matches.subcommand_matches("history") { return handle_history(&mut api, matches).await; - } else if let Some(matches) = matches.subcommand_matches("extensions") { - #[cfg(feature = "extensions")] - return handle_extensions(matches).await; } else if should_cancel { if let Some(matches) = matches.subcommand_matches("cancel") { let request_id = matches.value_of("request_id").unwrap().to_string(); @@ -217,6 +214,9 @@ async fn handle_commands() -> CommandResult { let resp = api.cancel(&request_id).await; print_response(&resp, true, None); } + } else if let Some(matches) = matches.subcommand_matches("extension") { + #[cfg(feature = "extensions")] + return handle_extensions(matches).await; } Ok(ExitCode::Ok.into()) diff --git a/cli/src/commands/extensions/extension.rs b/cli/src/commands/extensions/extension.rs index 77d5a7d6f..576720d95 100644 --- a/cli/src/commands/extensions/extension.rs +++ b/cli/src/commands/extensions/extension.rs @@ -53,6 +53,25 @@ impl Extension { Ok(()) } + pub fn uninstall(self) -> Result<()> { + let target_prefix = extensions_path()?.join(self.name()); + + if target_prefix != self.path { + return Err(anyhow!("extension is not installed, skipping")); + } + + for entry in walkdir::WalkDir::new(&self.path).contents_first(true) { + let entry = entry?.into_path(); + if entry.is_dir() { + std::fs::remove_dir(entry)?; + } else if entry.is_file() { + std::fs::remove_file(entry)?; + } + } + + Ok(()) + } + /// Load an extension from the default path. pub fn load(name: &str) -> Result { Extension::try_from(extensions_path()?.join(name)) diff --git a/cli/src/commands/extensions/mod.rs b/cli/src/commands/extensions/mod.rs index f8177f80a..77792024b 100644 --- a/cli/src/commands/extensions/mod.rs +++ b/cli/src/commands/extensions/mod.rs @@ -3,6 +3,7 @@ pub mod extension; use std::{convert::TryFrom, path::PathBuf}; pub use extension::*; +use log::info; use crate::commands::{CommandResult, CommandValue, ExitCode}; use anyhow::Result; @@ -56,6 +57,9 @@ pub async fn handle_extensions(matches: &ArgMatches) -> CommandResult { /// Add the extension from the specified path. pub async fn subcmd_add_extension(path: PathBuf) -> CommandResult { let extension = Extension::try_from(path)?; + info!("Installing extension {}...", extension.name()); + + extension.install()?; Ok(CommandValue::Code(ExitCode::Ok)) } @@ -63,7 +67,10 @@ pub async fn subcmd_add_extension(path: PathBuf) -> CommandResult { /// Handle the `extension remove` subcommand path. /// Remove the extension named as specified. pub async fn subcmd_remove_extension(name: &str) -> CommandResult { - let extension = Extension::try_from(extensions_path()?.join(name))?; + // let extension = Extension::try_from(extensions_path()?.join(name))?; + let extension = Extension::load(name)?; + + extension.uninstall()?; Ok(CommandValue::Code(ExitCode::Ok)) } diff --git a/cli/tests/extensions.rs b/cli/tests/extensions.rs index c059a02ee..bff59f2b5 100644 --- a/cli/tests/extensions.rs +++ b/cli/tests/extensions.rs @@ -1,27 +1,161 @@ use std::convert::TryFrom; +use std::ffi::OsStr; use std::path::{Path, PathBuf}; +use assert_cmd::Command; use phylum_cli::commands::extensions::*; use rand::Rng; +use regex::Regex; +//////////////////////////////////////////////////////////////////////////////// +// Acceptance criteria tests +//////////////////////////////////////////////////////////////////////////////// + +// When a user runs `phylum extension add .`, the extension in the current +// working directory should be installed. #[test] -fn valid_extension_is_loaded_correctly() { - let ext = Extension::try_from(fixtures_path().join("sample-extension")).unwrap(); +fn extension_is_installed_correctly() { + let tmp_dir = TmpDir::new(); + Command::cargo_bin("phylum") + .unwrap() + .env("XDG_DATA_HOME", &tmp_dir) + .arg("extension") + .arg("add") + .arg(fixtures_path().join("sample-extension")) + .assert(); - assert_eq!(ext.name(), "sample-extension"); + std::env::set_var("XDG_DATA_HOME", &tmp_dir); + + let installed_ext = Extension::load("sample-extension").unwrap(); + + assert_eq!(installed_ext.name(), "sample-extension"); + + let not_installed_ext = Extension::load("sample-other-extension"); + assert!(not_installed_ext.is_err()); } +// After a user installs a new extension, foobar, it should become available to +// the user under the phylum cli, e.g., running `phylum foobar` should execute +// the foobar extension. #[test] -fn extension_is_installed_correctly() { +fn can_run_installed_extension() { + let tmp_dir = TmpDir::new(); + Command::cargo_bin("phylum") + .unwrap() + .env("XDG_DATA_HOME", &tmp_dir) + .arg("extension") + .arg("add") + .arg(fixtures_path().join("sample-extension")) + .assert(); + + let cmd = Command::cargo_bin("phylum") + .unwrap() + .env("XDG_DATA_HOME", &tmp_dir) + .arg("sample-extension") + .assert(); + + cmd.success(); +} + +// When a user installs a valid extension it should print a message indicating +// success. It should also print a quick guide on the extension to give the user +// some context on how the given extension works. +#[test] +fn successful_installation_prints_message() { + todo!(); +} + +// When a user attempts to install an invalid extension, it should fail and +// inform the user as to why. +#[test] +fn unsuccessful_installation_prints_failure_message() { + todo!(); +} + +// When a user runs `phylum extension remove ` the extension +// should be entirely removed from the user system. +#[test] +fn extension_is_uninstalled_correctly() { + let tmp_dir = TmpDir::new(); + Command::cargo_bin("phylum") + .unwrap() + .env("XDG_DATA_HOME", &tmp_dir) + .arg("extension") + .arg("add") + .arg(fixtures_path().join("sample-extension")) + .assert(); + + assert!( + std::fs::read_dir(&tmp_dir) + .unwrap() + .into_iter() + .collect::>() + .len() + > 1 + ); + + Command::cargo_bin("phylum") + .unwrap() + .env("XDG_DATA_HOME", &tmp_dir) + .arg("extension") + .arg("remove") + .arg(fixtures_path().join("sample-extension")) + .assert(); + for i in std::fs::read_dir(&tmp_dir).unwrap() { + println!("{:?}", i); + } + assert!( + std::fs::read_dir(&tmp_dir) + .unwrap() + .into_iter() + .collect::>() + .len() + == 1 + ); +} + +// When a user runs phylum extension or phylum extension list a list of +// currently installed extensions, their versions and a short one sentence blurb +// on what the extension does should be shown in a table format. +#[test] +fn extension_list_should_emit_output() { let tmp_dir = TmpDir::new(); - std::env::set_var("XDG_DATA_HOME", tmp_dir.0.as_os_str()); + Command::cargo_bin("phylum") + .unwrap() + .env("XDG_DATA_HOME", &tmp_dir) + .arg("extension") + .arg("add") + .arg(fixtures_path().join("sample-extension")) + .assert(); + + let cmd = Command::cargo_bin("phylum") + .unwrap() + .env("XDG_DATA_HOME", &tmp_dir) + .arg("extension") + .arg("list") + .assert(); + + let output = std::str::from_utf8(&cmd.get_output().stdout).unwrap(); + let re = Regex::new(r#"^sample-extension\s+This extension does a thing"#).unwrap(); + + assert!(output.lines().find(|m| re.is_match(m)).is_some()); +} + +//////////////////////////////////////////////////////////////////////////////// +// Miscellaneous tests +//////////////////////////////////////////////////////////////////////////////// +#[test] +fn valid_extension_is_loaded_correctly() { let ext = Extension::try_from(fixtures_path().join("sample-extension")).unwrap(); - ext.install().unwrap(); - let installed_ext = Extension::load(ext.name()).unwrap(); + assert_eq!(ext.name(), "sample-extension"); } +//////////////////////////////////////////////////////////////////////////////// +// Utilities +//////////////////////////////////////////////////////////////////////////////// + fn project_root() -> PathBuf { Path::new(&env!("CARGO_MANIFEST_DIR")) .ancestors() @@ -60,9 +194,15 @@ impl AsRef for TmpDir { } } +impl AsRef for TmpDir { + fn as_ref(&self) -> &OsStr { + &self.0.as_os_str() + } +} + impl Drop for TmpDir { fn drop(&mut self) { - std::fs::remove_dir_all(&self.0).unwrap(); + // std::fs::remove_dir_all(&self.0).unwrap(); } } From 2e3fe42889dd062076faf3f131b00dac460a8cc5 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Thu, 5 May 2022 19:12:16 +0200 Subject: [PATCH 05/52] Acceptance criteria tests --- cli/src/app.rs | 1 + cli/src/commands/extensions/extension.rs | 10 +- cli/src/commands/extensions/mod.rs | 20 ++-- cli/tests/extensions.rs | 115 ++++++++++++++++++----- 4 files changed, 116 insertions(+), 30 deletions(-) diff --git a/cli/src/app.rs b/cli/src/app.rs index 71b6e24f3..9f5ecd021 100644 --- a/cli/src/app.rs +++ b/cli/src/app.rs @@ -141,6 +141,7 @@ pub fn app<'a>() -> clap::Command<'a> { #[cfg(feature = "extensions")] { app = app.subcommand(crate::commands::extensions::command()); + app = crate::commands::extensions::extensions_subcommands(app); } #[cfg(feature = "selfmanage")] diff --git a/cli/src/commands/extensions/extension.rs b/cli/src/commands/extensions/extension.rs index 576720d95..ad33ea01f 100644 --- a/cli/src/commands/extensions/extension.rs +++ b/cli/src/commands/extensions/extension.rs @@ -29,6 +29,7 @@ impl Extension { /// Install the extension in the default path. pub fn install(&self) -> Result<()> { + println!("Installing extension {}...", self.name()); let target_prefix = extensions_path()?.join(self.name()); if target_prefix == self.path { @@ -50,18 +51,23 @@ impl Extension { } } + println!("Extension {} installed correctly", self.name()); + Ok(()) } pub fn uninstall(self) -> Result<()> { + println!("Uninstalling extension {}...", self.name()); let target_prefix = extensions_path()?.join(self.name()); if target_prefix != self.path { - return Err(anyhow!("extension is not installed, skipping")); + println!("{:?} {:?}", target_prefix, self.path); + return Err(anyhow!("extension {} is not installed, skipping", self.name())); } for entry in walkdir::WalkDir::new(&self.path).contents_first(true) { let entry = entry?.into_path(); + println!("removing {}...", entry.to_string_lossy()); if entry.is_dir() { std::fs::remove_dir(entry)?; } else if entry.is_file() { @@ -69,6 +75,8 @@ impl Extension { } } + println!("Extension {} uninstalled correctly", self.name()); + Ok(()) } diff --git a/cli/src/commands/extensions/mod.rs b/cli/src/commands/extensions/mod.rs index 77792024b..980c82b9f 100644 --- a/cli/src/commands/extensions/mod.rs +++ b/cli/src/commands/extensions/mod.rs @@ -2,9 +2,9 @@ pub mod extension; use std::{convert::TryFrom, path::PathBuf}; -pub use extension::*; -use log::info; use crate::commands::{CommandResult, CommandValue, ExitCode}; +pub use extension::*; +use log::{error, info}; use anyhow::Result; use clap::{arg, ArgMatches, Command, ValueHint}; @@ -26,8 +26,16 @@ pub fn command<'a>() -> Command<'a> { } /// Generate the subcommands for each extension. -pub fn extensions_subcommands<'a>(command: Command<'a>) -> Result> { - let extensions = list_extensions()? +pub fn extensions_subcommands<'a>(command: Command<'a>) -> Command<'a> { + let extensions = match list_extensions() { + Ok(extensions) => extensions, + Err(e) => { + error!("Couldn't list extensions: {}", e); + return command; + } + }; + + let extensions = extensions .into_iter() .filter(|ext| { command @@ -36,9 +44,9 @@ pub fn extensions_subcommands<'a>(command: Command<'a>) -> Result> { }) .collect::>(); - Ok(extensions.into_iter().fold(command, |command, ext| { + extensions.into_iter().fold(command, |command, ext| { command.subcommand(Command::new(ext.name())) - })) + }) } /// Entry point for the `extensions` subcommand. diff --git a/cli/tests/extensions.rs b/cli/tests/extensions.rs index bff59f2b5..93bac089f 100644 --- a/cli/tests/extensions.rs +++ b/cli/tests/extensions.rs @@ -22,12 +22,12 @@ fn extension_is_installed_correctly() { .arg("extension") .arg("add") .arg(fixtures_path().join("sample-extension")) - .assert(); + .assert() + .success(); std::env::set_var("XDG_DATA_HOME", &tmp_dir); let installed_ext = Extension::load("sample-extension").unwrap(); - assert_eq!(installed_ext.name(), "sample-extension"); let not_installed_ext = Extension::load("sample-other-extension"); @@ -46,15 +46,25 @@ fn can_run_installed_extension() { .arg("extension") .arg("add") .arg(fixtures_path().join("sample-extension")) - .assert(); + .assert() + .success(); let cmd = Command::cargo_bin("phylum") .unwrap() .env("XDG_DATA_HOME", &tmp_dir) .arg("sample-extension") - .assert(); + .assert() + .success(); - cmd.success(); + // TODO match the output of the sample extension to ensure it executed properly. + // TODO build a proper fixture which prints some output. + let output = std::str::from_utf8(&cmd.get_output().stdout).unwrap(); + println!("{}", output); + todo!(r#" + We need to settle on an extension format + and build a test fixture before we can + allow this test to pass. + "#); } // When a user installs a valid extension it should print a message indicating @@ -62,14 +72,75 @@ fn can_run_installed_extension() { // some context on how the given extension works. #[test] fn successful_installation_prints_message() { - todo!(); + let tmp_dir = TmpDir::new(); + let cmd = Command::cargo_bin("phylum") + .unwrap() + .env("XDG_DATA_HOME", &tmp_dir) + .arg("extension") + .arg("add") + .arg(fixtures_path().join("sample-extension")) + .assert() + .success(); + + let output = std::str::from_utf8(&cmd.get_output().stdout).unwrap(); + let re = Regex::new(r#"^Extension sample-extension installed correctly"#).unwrap(); + + assert!(output.lines().find(|m| re.is_match(m)).is_some()); } // When a user attempts to install an invalid extension, it should fail and // inform the user as to why. #[test] fn unsuccessful_installation_prints_failure_message() { - todo!(); + let tmp_dir = TmpDir::new(); + + fn stderr_match_regex(cmd: assert_cmd::assert::Assert, pattern: &str) -> bool { + let output = std::str::from_utf8(&cmd.get_output().stderr).unwrap(); + let re = Regex::new(pattern).unwrap(); + + output.lines().find(|m| re.is_match(m)).is_some() + } + + // Install the extension. Should succeed. + Command::cargo_bin("phylum") + .unwrap() + .env("XDG_DATA_HOME", &tmp_dir) + .arg("extension") + .arg("add") + .arg(fixtures_path().join("sample-extension")) + .assert() + .success(); + + // Reinstall the same extension. Should fail with an error. + assert!(stderr_match_regex( + Command::cargo_bin("phylum") + .unwrap() + .env("XDG_DATA_HOME", &tmp_dir) + .arg("extension") + .arg("add") + .arg(fixtures_path().join("sample-extension")) + .assert() + .failure(), + r#": already exists"#, + )); + + // Try to install the extension from the installed path. Should fail with an error. + assert!(stderr_match_regex( + Command::cargo_bin("phylum") + .unwrap() + .env("XDG_DATA_HOME", &tmp_dir) + .arg("extension") + .arg("add") + .arg( + PathBuf::from(&tmp_dir) + .join("phylum") + .join("extensions") + .join("sample-extension"), + ) + .assert() + .failure(), + "skipping", + )); } // When a user runs `phylum extension remove ` the extension @@ -77,17 +148,23 @@ fn unsuccessful_installation_prints_failure_message() { #[test] fn extension_is_uninstalled_correctly() { let tmp_dir = TmpDir::new(); + Command::cargo_bin("phylum") .unwrap() .env("XDG_DATA_HOME", &tmp_dir) .arg("extension") .arg("add") .arg(fixtures_path().join("sample-extension")) - .assert(); + .assert() + .success(); + + let extension_path = PathBuf::from(&tmp_dir) + .join("phylum") + .join("extensions") + .join("sample-extension"); assert!( - std::fs::read_dir(&tmp_dir) - .unwrap() + walkdir::WalkDir::new(&extension_path) .into_iter() .collect::>() .len() @@ -99,19 +176,11 @@ fn extension_is_uninstalled_correctly() { .env("XDG_DATA_HOME", &tmp_dir) .arg("extension") .arg("remove") - .arg(fixtures_path().join("sample-extension")) - .assert(); - for i in std::fs::read_dir(&tmp_dir).unwrap() { - println!("{:?}", i); - } - assert!( - std::fs::read_dir(&tmp_dir) - .unwrap() - .into_iter() - .collect::>() - .len() - == 1 - ); + .arg("sample-extension") + .assert() + .success(); + + assert!(!extension_path.exists()); } // When a user runs phylum extension or phylum extension list a list of From e0b2d1721a423b77f53e851e6c9f4479a93c4e23 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Fri, 6 May 2022 14:14:33 +0200 Subject: [PATCH 06/52] Acceptance criteria tests --- cli/src/commands/extensions/mod.rs | 18 ++++++++++++++---- cli/tests/extensions.rs | 16 ++++++++++++++++ 2 files changed, 30 insertions(+), 4 deletions(-) diff --git a/cli/src/commands/extensions/mod.rs b/cli/src/commands/extensions/mod.rs index 980c82b9f..1cbabca13 100644 --- a/cli/src/commands/extensions/mod.rs +++ b/cli/src/commands/extensions/mod.rs @@ -88,16 +88,26 @@ pub async fn subcmd_remove_extension(name: &str) -> CommandResult { pub async fn subcmd_list_extensions() -> CommandResult { let extensions = list_extensions()?; - extensions.into_iter().for_each(|ext| { - println!("{:20} {}", ext.name(), ext.description().unwrap_or("")); - }); + if extensions.is_empty() { + println!("No extensions are currently installed."); + } else { + extensions.into_iter().for_each(|ext| { + println!("{:20} {}", ext.name(), ext.description().unwrap_or("")); + }); + } Ok(CommandValue::Code(ExitCode::Ok)) } // Return a list of installed extensions. Filter out invalid extensions instead of exiting early. fn list_extensions() -> Result> { - Ok(std::fs::read_dir(extensions_path()?)? + let extension_path = extensions_path()?; + + if !extension_path.exists() { + return Ok(Vec::new()); + } + + Ok(std::fs::read_dir(extension_path)? .filter_map(|d| Extension::try_from(d.map(|d| d.path()).ok()?).ok()) .collect::>()) } diff --git a/cli/tests/extensions.rs b/cli/tests/extensions.rs index 93bac089f..ae0574063 100644 --- a/cli/tests/extensions.rs +++ b/cli/tests/extensions.rs @@ -189,6 +189,21 @@ fn extension_is_uninstalled_correctly() { #[test] fn extension_list_should_emit_output() { let tmp_dir = TmpDir::new(); + + // Output that no extension is installed when that is the case. + let cmd = Command::cargo_bin("phylum") + .unwrap() + .env("XDG_DATA_HOME", &tmp_dir) + .arg("extension") + .arg("list") + .assert(); + + let output = std::str::from_utf8(&cmd.get_output().stdout).unwrap(); + let re = Regex::new(r#"No extension"#).unwrap(); + + assert!(output.lines().find(|m| re.is_match(m)).is_some()); + + // Install one extension Command::cargo_bin("phylum") .unwrap() .env("XDG_DATA_HOME", &tmp_dir) @@ -197,6 +212,7 @@ fn extension_list_should_emit_output() { .arg(fixtures_path().join("sample-extension")) .assert(); + // Output name and description of the extension when one is installed let cmd = Command::cargo_bin("phylum") .unwrap() .env("XDG_DATA_HOME", &tmp_dir) From 5bd4ec8e23af9b9e49637548942641d436600910 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Fri, 6 May 2022 14:42:17 +0200 Subject: [PATCH 07/52] Extensions base documentation --- docs/extensions.md | 50 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) create mode 100644 docs/extensions.md diff --git a/docs/extensions.md b/docs/extensions.md new file mode 100644 index 000000000..7c8ee3b1b --- /dev/null +++ b/docs/extensions.md @@ -0,0 +1,50 @@ +--- +title: Extensions +category: 61e72e3a50a88e001a92ee5d +--- + +It is possible to extend the Phylum CLI with external sub-commands. + +To install an extension, run the following command: + +```sh +phylum extension add path/to/extension +``` + +The extension will be installed under `$XDG_DATA_HOME/phylum/extensions/`. + +Once installed, the extension will be accessible via the Phylum CLI: + +```sh +phylum [arguments...] +``` + +To list the currently installed extension, run the following command: + +```sh +phylum extension list +``` + +To uninstall a previously installed extension, run the following command: + +```sh +phylum extension remove +``` + +## Extension format + +**TODO**: Rectify this section once the final decisions on the format have been taken. + +An extension is comprised of a directory containing a *manifest file*, named +`PhylumExt.toml`, an executable file (the *entry point*) and as many auxilliary +files as the extension may require. The `PhylumExt.toml` manifest is structured +this way: + +```toml +name = "extension-name" +description = "Brief description of the extension" +entry_point = "sample_extension.sh" +``` + +The *entry point* is the executable that will be run via `phylum extension-name`. +All arguments passed via the CLI will be forwarded to the extension executable. From 3f059a3623c376f275ee4d5c1ea69098fa3c8437 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Fri, 6 May 2022 15:10:07 +0200 Subject: [PATCH 08/52] Clippy lints, conditional compilation fix for compiling with no features --- cli/src/bin/phylum.rs | 6 ++++-- cli/src/commands/extensions/extension.rs | 2 +- cli/src/commands/extensions/mod.rs | 24 ++++++++++++------------ 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/cli/src/bin/phylum.rs b/cli/src/bin/phylum.rs index aa3257e34..33ee97e7a 100644 --- a/cli/src/bin/phylum.rs +++ b/cli/src/bin/phylum.rs @@ -214,9 +214,11 @@ async fn handle_commands() -> CommandResult { let resp = api.cancel(&request_id).await; print_response(&resp, true, None); } - } else if let Some(matches) = matches.subcommand_matches("extension") { + } else if cfg!(feature = "extensions") { #[cfg(feature = "extensions")] - return handle_extensions(matches).await; + if let Some(matches) = matches.subcommand_matches("extension") { + return handle_extensions(matches).await; + } } Ok(ExitCode::Ok.into()) diff --git a/cli/src/commands/extensions/extension.rs b/cli/src/commands/extensions/extension.rs index ad33ea01f..ff567548e 100644 --- a/cli/src/commands/extensions/extension.rs +++ b/cli/src/commands/extensions/extension.rs @@ -24,7 +24,7 @@ impl Extension { } pub fn description(&self) -> Option<&str> { - self.manifest.description.as_ref().map(String::as_str) + self.manifest.description.as_deref() } /// Install the extension in the default path. diff --git a/cli/src/commands/extensions/mod.rs b/cli/src/commands/extensions/mod.rs index 1cbabca13..fce6d9574 100644 --- a/cli/src/commands/extensions/mod.rs +++ b/cli/src/commands/extensions/mod.rs @@ -1,6 +1,6 @@ pub mod extension; -use std::{convert::TryFrom, path::PathBuf}; +use std::{collections::HashSet, convert::TryFrom, path::PathBuf}; use crate::commands::{CommandResult, CommandValue, ExitCode}; pub use extension::*; @@ -26,7 +26,8 @@ pub fn command<'a>() -> Command<'a> { } /// Generate the subcommands for each extension. -pub fn extensions_subcommands<'a>(command: Command<'a>) -> Command<'a> { +/// TODO add tests. +pub fn extensions_subcommands(command: Command<'_>) -> Command<'_> { let extensions = match list_extensions() { Ok(extensions) => extensions, Err(e) => { @@ -35,18 +36,17 @@ pub fn extensions_subcommands<'a>(command: Command<'a>) -> Command<'a> { } }; - let extensions = extensions + let names = command + .get_subcommands() + .map(|n| n.get_name().to_string()) + .collect::>(); + + extensions .into_iter() - .filter(|ext| { - command - .get_subcommands() - .all(|p| p.get_name() != ext.name()) + .filter(|ext| !names.contains(ext.name())) + .fold(command, |command, ext| { + command.subcommand(Command::new(ext.name())) }) - .collect::>(); - - extensions.into_iter().fold(command, |command, ext| { - command.subcommand(Command::new(ext.name())) - }) } /// Entry point for the `extensions` subcommand. From e54eddcbea6d0c6457521e1937df61ae4150b645 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Fri, 6 May 2022 15:25:14 +0200 Subject: [PATCH 09/52] Cargo fmt --- cli/src/commands/extensions/extension.rs | 10 +++++++--- cli/src/commands/mod.rs | 4 ++-- cli/tests/extensions.rs | 6 ++++-- 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/cli/src/commands/extensions/extension.rs b/cli/src/commands/extensions/extension.rs index ff567548e..52c2c8501 100644 --- a/cli/src/commands/extensions/extension.rs +++ b/cli/src/commands/extensions/extension.rs @@ -33,7 +33,9 @@ impl Extension { let target_prefix = extensions_path()?.join(self.name()); if target_prefix == self.path { - return Err(anyhow!("extension path and installation path are identical, skipping")); + return Err(anyhow!( + "extension path and installation path are identical, skipping" + )); } for entry in walkdir::WalkDir::new(&self.path) { @@ -62,7 +64,10 @@ impl Extension { if target_prefix != self.path { println!("{:?} {:?}", target_prefix, self.path); - return Err(anyhow!("extension {} is not installed, skipping", self.name())); + return Err(anyhow!( + "extension {} is not installed, skipping", + self.name() + )); } for entry in walkdir::WalkDir::new(&self.path).contents_first(true) { @@ -132,7 +137,6 @@ impl TryFrom for Extension { } } - // Construct and return the extension path: $XDG_DATA_HOME/phylum/extensions pub fn extensions_path() -> Result { Ok(crate::config::data_dir()?.join("phylum").join("extensions")) diff --git a/cli/src/commands/mod.rs b/cli/src/commands/mod.rs index 158784f02..a6838afc6 100644 --- a/cli/src/commands/mod.rs +++ b/cli/src/commands/mod.rs @@ -1,12 +1,12 @@ use phylum_types::types::job::Action; pub mod auth; +#[cfg(feature = "extensions")] +pub mod extensions; pub mod jobs; pub mod lock_files; pub mod packages; pub mod project; -#[cfg(feature = "extensions")] -pub mod extensions; #[cfg(feature = "selfmanage")] pub mod uninstall; diff --git a/cli/tests/extensions.rs b/cli/tests/extensions.rs index ae0574063..92a031d5e 100644 --- a/cli/tests/extensions.rs +++ b/cli/tests/extensions.rs @@ -60,11 +60,13 @@ fn can_run_installed_extension() { // TODO build a proper fixture which prints some output. let output = std::str::from_utf8(&cmd.get_output().stdout).unwrap(); println!("{}", output); - todo!(r#" + todo!( + r#" We need to settle on an extension format and build a test fixture before we can allow this test to pass. - "#); + "# + ); } // When a user installs a valid extension it should print a message indicating From a13e64b7ea3a62a7a6fd52d4978cbab57312c859 Mon Sep 17 00:00:00 2001 From: andreaphylum <68552656+andreaphylum@users.noreply.github.com> Date: Fri, 6 May 2022 16:58:10 +0200 Subject: [PATCH 10/52] Update cli/Cargo.toml Co-authored-by: Kyle Willmon --- cli/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 223b09bc5..2704c8cb3 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -5,7 +5,7 @@ authors = ["Eric Freitag "] edition = "2018" [features] -default = ["selfmanage", "extensions"] +default = ["selfmanage"] phylum-online = [] selfmanage = [] extensions = [] From 5f5105c82355878bba04a1f1adb221e419a34da7 Mon Sep 17 00:00:00 2001 From: andreaphylum <68552656+andreaphylum@users.noreply.github.com> Date: Fri, 6 May 2022 17:03:13 +0200 Subject: [PATCH 11/52] Update cli/src/commands/extensions/mod.rs Co-authored-by: Kyle Willmon --- cli/src/commands/extensions/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/src/commands/extensions/mod.rs b/cli/src/commands/extensions/mod.rs index fce6d9574..97f066e17 100644 --- a/cli/src/commands/extensions/mod.rs +++ b/cli/src/commands/extensions/mod.rs @@ -15,7 +15,7 @@ pub fn command<'a>() -> Command<'a> { .subcommand( Command::new("add") .about("Install extension") - .arg(arg!([PATH]).required(true).value_hint(ValueHint::FilePath)), + .arg(arg!([PATH]).required(true).value_hint(ValueHint::DirPath)), ) .subcommand( Command::new("remove") From 3140baff59eb60f2193417b3baa45208dd9aee39 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Fri, 6 May 2022 19:44:49 +0200 Subject: [PATCH 12/52] Review fixes --- cli/src/app.rs | 7 +- cli/src/bin/phylum.rs | 10 +-- cli/src/commands/extensions/extension.rs | 66 +++++++++++++------ cli/src/commands/extensions/mod.rs | 55 +++++++++++----- cli/tests/extensions.rs | 55 +++++++++++----- .../extensions/ping-extension/PhylumExt.toml | 3 + .../ping-extension/sample_extension.sh | 3 + 7 files changed, 136 insertions(+), 63 deletions(-) create mode 100644 cli/tests/fixtures/extensions/ping-extension/PhylumExt.toml create mode 100644 cli/tests/fixtures/extensions/ping-extension/sample_extension.sh diff --git a/cli/src/app.rs b/cli/src/app.rs index 9f5ecd021..2391c2f41 100644 --- a/cli/src/app.rs +++ b/cli/src/app.rs @@ -1,6 +1,9 @@ use clap::{arg, Command, ValueHint}; use git_version::git_version; +#[cfg(feature = "extensions")] +use crate::commands::extensions; + const VERSION: &str = git_version!( args = ["--dirty=-modified", "--tags"], cargo_prefix = "cargo:" @@ -140,8 +143,8 @@ pub fn app<'a>() -> clap::Command<'a> { #[cfg(feature = "extensions")] { - app = app.subcommand(crate::commands::extensions::command()); - app = crate::commands::extensions::extensions_subcommands(app); + app = app.subcommand(extensions::command()); + app = extensions::extensions_subcommands(app); } #[cfg(feature = "selfmanage")] diff --git a/cli/src/bin/phylum.rs b/cli/src/bin/phylum.rs index 33ee97e7a..28a700b7d 100644 --- a/cli/src/bin/phylum.rs +++ b/cli/src/bin/phylum.rs @@ -64,6 +64,11 @@ async fn handle_commands() -> CommandResult { return handle_uninstall(matches); } + #[cfg(feature = "extensions")] + if let Some(matches) = matches.subcommand_matches("extension") { + return handle_extensions(matches).await; + } + let settings_path = get_home_settings_path()?; let config_path = matches @@ -214,11 +219,6 @@ async fn handle_commands() -> CommandResult { let resp = api.cancel(&request_id).await; print_response(&resp, true, None); } - } else if cfg!(feature = "extensions") { - #[cfg(feature = "extensions")] - if let Some(matches) = matches.subcommand_matches("extension") { - return handle_extensions(matches).await; - } } Ok(ExitCode::Ok.into()) diff --git a/cli/src/commands/extensions/extension.rs b/cli/src/commands/extensions/extension.rs index 52c2c8501..ee5684ec6 100644 --- a/cli/src/commands/extensions/extension.rs +++ b/cli/src/commands/extensions/extension.rs @@ -1,10 +1,22 @@ -use std::{convert::TryFrom, fs::File, io::Read, path::PathBuf}; +use std::convert::TryFrom; +use std::fs::{self, File, Permissions}; +use std::io::Read; +use std::path::PathBuf; +#[cfg(unix)] +use std::os::unix::prelude::PermissionsExt; use anyhow::{anyhow, Result}; +use lazy_static::lazy_static; +use regex::Regex; use serde::Deserialize; +use walkdir::WalkDir; const MANIFEST_NAME: &str = "PhylumExt.toml"; +lazy_static! { + static ref EXTENSION_NAME_RE: Regex = Regex::new(r#"^[a-z0-9_-]+$"#).unwrap(); +} + #[derive(Debug)] pub struct Extension { path: PathBuf, @@ -30,7 +42,14 @@ impl Extension { /// Install the extension in the default path. pub fn install(&self) -> Result<()> { println!("Installing extension {}...", self.name()); - let target_prefix = extensions_path()?.join(self.name()); + + let target_prefix = extension_path(self.name())?; + + // TODO we may want to implement `upgrade` in the future, which would + // allow writing to the path of an already installed extension. + if target_prefix.exists() { + return Err(anyhow!("extension already exists, skipping")); + } if target_prefix == self.path { return Err(anyhow!( @@ -38,56 +57,49 @@ impl Extension { )); } - for entry in walkdir::WalkDir::new(&self.path) { + for entry in WalkDir::new(&self.path) { let source_path = entry?.into_path(); let dest_path = target_prefix.join(source_path.strip_prefix(&self.path)?); if source_path.is_dir() { - std::fs::create_dir_all(dest_path)?; + fs::create_dir_all(&dest_path)?; + #[cfg(unix)] + fs::set_permissions(&dest_path, Permissions::from_mode(0o700))?; } else if source_path.is_file() { if dest_path.exists() { return Err(anyhow!("{}: already exists", dest_path.to_string_lossy())); } else { - std::fs::copy(source_path, dest_path)?; + fs::copy(source_path, dest_path)?; } } } - println!("Extension {} installed correctly", self.name()); + println!("Extension {} installed successfully", self.name()); Ok(()) } pub fn uninstall(self) -> Result<()> { println!("Uninstalling extension {}...", self.name()); - let target_prefix = extensions_path()?.join(self.name()); + let target_prefix = extension_path(self.name())?; if target_prefix != self.path { - println!("{:?} {:?}", target_prefix, self.path); return Err(anyhow!( "extension {} is not installed, skipping", self.name() )); } - for entry in walkdir::WalkDir::new(&self.path).contents_first(true) { - let entry = entry?.into_path(); - println!("removing {}...", entry.to_string_lossy()); - if entry.is_dir() { - std::fs::remove_dir(entry)?; - } else if entry.is_file() { - std::fs::remove_file(entry)?; - } - } + fs::remove_dir_all(&self.path)?; - println!("Extension {} uninstalled correctly", self.name()); + println!("Extension {} uninstalled successfully", self.name()); Ok(()) } /// Load an extension from the default path. pub fn load(name: &str) -> Result { - Extension::try_from(extensions_path()?.join(name)) + Extension::try_from(extension_path(name)?) } } @@ -129,8 +141,14 @@ impl TryFrom for Extension { )); } + if !EXTENSION_NAME_RE.is_match(&manifest.name) { + return Err(anyhow!( + "{}: invalid extension name, must be lowercase alphanumeric, dash (-) or underscore (_)", + manifest.name + )); + } + // TODO add further validation if necessary: - // - Check that the name matches /^[a-z0-9-_]+$/ // - Check that the entry point is a supported format (.wasm?) // - Check that the entry point is appropriately signed Ok(Extension { path, manifest }) @@ -141,3 +159,11 @@ impl TryFrom for Extension { pub fn extensions_path() -> Result { Ok(crate::config::data_dir()?.join("phylum").join("extensions")) } + +pub fn extension_path(name: &str) -> Result { + if !EXTENSION_NAME_RE.is_match(name) { + return Err(anyhow!("{}: invalid extension name, must be lowercase alphanumeric, dash (-) or underscore (_) ", name)); + } + + Ok(extensions_path()?.join(name)) +} diff --git a/cli/src/commands/extensions/mod.rs b/cli/src/commands/extensions/mod.rs index fce6d9574..3311cae17 100644 --- a/cli/src/commands/extensions/mod.rs +++ b/cli/src/commands/extensions/mod.rs @@ -4,10 +4,10 @@ use std::{collections::HashSet, convert::TryFrom, path::PathBuf}; use crate::commands::{CommandResult, CommandValue, ExitCode}; pub use extension::*; -use log::{error, info}; -use anyhow::Result; +use anyhow::{anyhow, Result}; use clap::{arg, ArgMatches, Command, ValueHint}; +use log::{error, warn}; pub fn command<'a>() -> Command<'a> { Command::new("extension") @@ -28,7 +28,7 @@ pub fn command<'a>() -> Command<'a> { /// Generate the subcommands for each extension. /// TODO add tests. pub fn extensions_subcommands(command: Command<'_>) -> Command<'_> { - let extensions = match list_extensions() { + let extensions = match installed_extensions() { Ok(extensions) => extensions, Err(e) => { error!("Couldn't list extensions: {}", e); @@ -43,7 +43,17 @@ pub fn extensions_subcommands(command: Command<'_>) -> Command<'_> { extensions .into_iter() - .filter(|ext| !names.contains(ext.name())) + .filter(|ext| { + if names.contains(ext.name()) { + warn!( + "{}: extension was filtered out due to name conflict", + ext.name() + ); + false + } else { + true + } + }) .fold(command, |command, ext| { command.subcommand(Command::new(ext.name())) }) @@ -52,20 +62,19 @@ pub fn extensions_subcommands(command: Command<'_>) -> Command<'_> { /// Entry point for the `extensions` subcommand. pub async fn handle_extensions(matches: &ArgMatches) -> CommandResult { if let Some(matches) = matches.subcommand_matches("add") { - subcmd_add_extension(matches.value_of_t("PATH")?).await + handle_add_extension(matches.value_of_t("PATH")?).await } else if let Some(matches) = matches.subcommand_matches("remove") { - subcmd_remove_extension(matches.value_of("NAME").unwrap()).await + handle_remove_extension(matches.value_of("NAME").unwrap()).await } else { // also covers the `list` case - subcmd_list_extensions().await + handle_list_extensions().await } } /// Handle the `extension add` subcommand path. /// Add the extension from the specified path. -pub async fn subcmd_add_extension(path: PathBuf) -> CommandResult { +async fn handle_add_extension(path: PathBuf) -> CommandResult { let extension = Extension::try_from(path)?; - info!("Installing extension {}...", extension.name()); extension.install()?; @@ -74,8 +83,7 @@ pub async fn subcmd_add_extension(path: PathBuf) -> CommandResult { /// Handle the `extension remove` subcommand path. /// Remove the extension named as specified. -pub async fn subcmd_remove_extension(name: &str) -> CommandResult { - // let extension = Extension::try_from(extensions_path()?.join(name))?; +async fn handle_remove_extension(name: &str) -> CommandResult { let extension = Extension::load(name)?; extension.uninstall()?; @@ -85,8 +93,8 @@ pub async fn subcmd_remove_extension(name: &str) -> CommandResult { /// Handle the `extension` / `extension list` subcommand path. /// List installed extensions. -pub async fn subcmd_list_extensions() -> CommandResult { - let extensions = list_extensions()?; +async fn handle_list_extensions() -> CommandResult { + let extensions = installed_extensions()?; if extensions.is_empty() { println!("No extensions are currently installed."); @@ -100,14 +108,25 @@ pub async fn subcmd_list_extensions() -> CommandResult { } // Return a list of installed extensions. Filter out invalid extensions instead of exiting early. -fn list_extensions() -> Result> { - let extension_path = extensions_path()?; +fn installed_extensions() -> Result> { + let extensions_path = extensions_path()?; - if !extension_path.exists() { + if !extensions_path.exists() { return Ok(Vec::new()); } - Ok(std::fs::read_dir(extension_path)? - .filter_map(|d| Extension::try_from(d.map(|d| d.path()).ok()?).ok()) + Ok(std::fs::read_dir(extensions_path)? + .filter_map(|dir_entry| { + match dir_entry + .map_err(|e| anyhow!("{}", e)) + .and_then(|dir_entry| Extension::try_from(dir_entry.path())) + { + Ok(ext) => Some(ext), + Err(e) => { + error!("{e}"); + None + } + } + }) .collect::>()) } diff --git a/cli/tests/extensions.rs b/cli/tests/extensions.rs index 92a031d5e..9aa3e0105 100644 --- a/cli/tests/extensions.rs +++ b/cli/tests/extensions.rs @@ -1,4 +1,6 @@ use std::convert::TryFrom; +use std::env; +use std::fs; use std::ffi::OsStr; use std::path::{Path, PathBuf}; @@ -25,7 +27,7 @@ fn extension_is_installed_correctly() { .assert() .success(); - std::env::set_var("XDG_DATA_HOME", &tmp_dir); + env::set_var("XDG_DATA_HOME", &tmp_dir); let installed_ext = Extension::load("sample-extension").unwrap(); assert_eq!(installed_ext.name(), "sample-extension"); @@ -56,17 +58,10 @@ fn can_run_installed_extension() { .assert() .success(); - // TODO match the output of the sample extension to ensure it executed properly. + // TODO match the output of the sample extension to ensure it executed properly. #357 // TODO build a proper fixture which prints some output. let output = std::str::from_utf8(&cmd.get_output().stdout).unwrap(); println!("{}", output); - todo!( - r#" - We need to settle on an extension format - and build a test fixture before we can - allow this test to pass. - "# - ); } // When a user installs a valid extension it should print a message indicating @@ -85,9 +80,7 @@ fn successful_installation_prints_message() { .success(); let output = std::str::from_utf8(&cmd.get_output().stdout).unwrap(); - let re = Regex::new(r#"^Extension sample-extension installed correctly"#).unwrap(); - - assert!(output.lines().find(|m| re.is_match(m)).is_some()); + assert!(output.lines().find(|m| m.contains("Extension sample-extension installed successfully")).is_some()); } // When a user attempts to install an invalid extension, it should fail and @@ -98,9 +91,8 @@ fn unsuccessful_installation_prints_failure_message() { fn stderr_match_regex(cmd: assert_cmd::assert::Assert, pattern: &str) -> bool { let output = std::str::from_utf8(&cmd.get_output().stderr).unwrap(); - let re = Regex::new(pattern).unwrap(); - output.lines().find(|m| re.is_match(m)).is_some() + output.lines().find(|m| m.contains(pattern)).is_some() } // Install the extension. Should succeed. @@ -123,7 +115,7 @@ fn unsuccessful_installation_prints_failure_message() { .arg(fixtures_path().join("sample-extension")) .assert() .failure(), - r#": already exists"#, + r#"extension already exists"#, )); // Try to install the extension from the installed path. Should fail with an error. @@ -239,6 +231,31 @@ fn valid_extension_is_loaded_correctly() { assert_eq!(ext.name(), "sample-extension"); } +#[test] +fn conflicting_extension_name_is_filtered() { + let tmp_dir = TmpDir::new(); + + Command::cargo_bin("phylum") + .unwrap() + .env("XDG_DATA_HOME", &tmp_dir) + .arg("extension") + .arg("add") + .arg(fixtures_path().join("ping-extension")) + .assert() + .success(); + + let cmd = Command::cargo_bin("phylum") + .unwrap() + .env("XDG_DATA_HOME", &tmp_dir) + .arg("extension") + .arg("list") + .assert() + .success(); + + let output = std::str::from_utf8(&cmd.get_output().stderr).unwrap(); + assert!(output.contains("extension was filtered out")); +} + //////////////////////////////////////////////////////////////////////////////// // Utilities //////////////////////////////////////////////////////////////////////////////// @@ -263,14 +280,16 @@ struct TmpDir(PathBuf); impl TmpDir { fn new() -> Self { - let dir_name: String = rand::thread_rng() + let dir_uid: String = rand::thread_rng() .sample_iter(&rand::distributions::Alphanumeric) .take(16) .map(char::from) .collect(); + let dir_name = format!("phylum-test-{dir_uid}"); + let path = tmp_path().join(dir_name); - std::fs::create_dir_all(&path).unwrap(); + fs::create_dir_all(&path).unwrap(); Self(path) } } @@ -289,7 +308,7 @@ impl AsRef for TmpDir { impl Drop for TmpDir { fn drop(&mut self) { - // std::fs::remove_dir_all(&self.0).unwrap(); + // fs::remove_dir_all(&self.0).unwrap(); } } diff --git a/cli/tests/fixtures/extensions/ping-extension/PhylumExt.toml b/cli/tests/fixtures/extensions/ping-extension/PhylumExt.toml new file mode 100644 index 000000000..c86a588ea --- /dev/null +++ b/cli/tests/fixtures/extensions/ping-extension/PhylumExt.toml @@ -0,0 +1,3 @@ +name = "ping" +description = "This extension should be filtered out" +entry_point = "sample_extension.sh" diff --git a/cli/tests/fixtures/extensions/ping-extension/sample_extension.sh b/cli/tests/fixtures/extensions/ping-extension/sample_extension.sh new file mode 100644 index 000000000..facae01d1 --- /dev/null +++ b/cli/tests/fixtures/extensions/ping-extension/sample_extension.sh @@ -0,0 +1,3 @@ +#!/bin/sh + +echo "Hello, world!" From aa4adc2e740242b2dbc1842341af0a589515b921 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Fri, 6 May 2022 20:00:55 +0200 Subject: [PATCH 13/52] Review fixes --- cli/src/config.rs | 13 ++++++++----- cli/tests/extensions.rs | 2 ++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/cli/src/config.rs b/cli/src/config.rs index 567a31a1c..cc653449d 100644 --- a/cli/src/config.rs +++ b/cli/src/config.rs @@ -196,14 +196,17 @@ pub fn get_home_settings_path() -> Result { /// Resolve XDG data directory. pub fn data_dir() -> Result { - let home_path = - home::home_dir().ok_or_else(|| anyhow!("Couldn't find the user's home directory"))?; - - Ok(env::var("XDG_DATA_HOME") + if let Some(data_dir) = env::var("XDG_DATA_HOME") .ok() .filter(|s| !s.is_empty()) .map(PathBuf::from) - .unwrap_or_else(|| home_path.join(".local").join("share"))) + { + Ok(data_dir) + } else { + home::home_dir() + .ok_or_else(|| anyhow!("Couldn't find the user's home directory")) + .map(|home_path| home_path.join(".local").join("share")) + } } /// Resolve XDG config directory. diff --git a/cli/tests/extensions.rs b/cli/tests/extensions.rs index 9aa3e0105..3dc24f422 100644 --- a/cli/tests/extensions.rs +++ b/cli/tests/extensions.rs @@ -1,3 +1,5 @@ +#![cfg(feature = "extensions")] + use std::convert::TryFrom; use std::env; use std::fs; From 5ee23e784a91afc12c63871424a8708d474bada4 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Mon, 9 May 2022 16:01:28 +0200 Subject: [PATCH 14/52] Documentation with new format --- docs/extensions.md | 33 +++------------------------------ docs/phylum_extension_add.md | 17 +++++++++++++++++ docs/phylum_extension_list.md | 10 ++++++++++ docs/phylum_extension_remove.md | 10 ++++++++++ 4 files changed, 40 insertions(+), 30 deletions(-) create mode 100644 docs/phylum_extension_add.md create mode 100644 docs/phylum_extension_list.md create mode 100644 docs/phylum_extension_remove.md diff --git a/docs/extensions.md b/docs/extensions.md index 7c8ee3b1b..eeb9a67ee 100644 --- a/docs/extensions.md +++ b/docs/extensions.md @@ -1,36 +1,9 @@ --- -title: Extensions -category: 61e72e3a50a88e001a92ee5d +title: Extension format +category: 6255e67693d5200013b1fa41 +hidden: true --- - -It is possible to extend the Phylum CLI with external sub-commands. - -To install an extension, run the following command: - -```sh -phylum extension add path/to/extension -``` - -The extension will be installed under `$XDG_DATA_HOME/phylum/extensions/`. - -Once installed, the extension will be accessible via the Phylum CLI: - -```sh -phylum [arguments...] ``` - -To list the currently installed extension, run the following command: - -```sh -phylum extension list -``` - -To uninstall a previously installed extension, run the following command: - -```sh -phylum extension remove -``` - ## Extension format **TODO**: Rectify this section once the final decisions on the format have been taken. diff --git a/docs/phylum_extension_add.md b/docs/phylum_extension_add.md new file mode 100644 index 000000000..59933c0ef --- /dev/null +++ b/docs/phylum_extension_add.md @@ -0,0 +1,17 @@ +--- +title: phylum extension add +category: 6255e67693d5200013b1fa3e +hidden: true +--- +To install an extension, run the following command: + +```sh +phylum extension add path/to/extension +``` + +The extension will be installed under `$XDG_DATA_HOME/phylum/extensions/`. +Once installed, the extension will be accessible via the Phylum CLI: + +```sh +phylum [arguments...] +``` diff --git a/docs/phylum_extension_list.md b/docs/phylum_extension_list.md new file mode 100644 index 000000000..40d9c6546 --- /dev/null +++ b/docs/phylum_extension_list.md @@ -0,0 +1,10 @@ +--- +title: phylum extension list +category: 6255e67693d5200013b1fa3e +hidden: true +--- +To list the currently installed extensions, run the following command: + +```sh +phylum extension list +``` diff --git a/docs/phylum_extension_remove.md b/docs/phylum_extension_remove.md new file mode 100644 index 000000000..4b09cbb36 --- /dev/null +++ b/docs/phylum_extension_remove.md @@ -0,0 +1,10 @@ +--- +title: phylum extension remove +category: 6255e67693d5200013b1fa3e +hidden: true +--- +To uninstall a previously installed extension, run the following command: + +```sh +phylum extension remove +``` From 88d23d4cad61f8efed82924a1e2ae459f6f43f65 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Mon, 9 May 2022 16:29:13 +0200 Subject: [PATCH 15/52] Env var cleanup skip; cargo fmt --- cli/src/commands/extensions/extension.rs | 2 +- cli/tests/extensions.rs | 11 ++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/cli/src/commands/extensions/extension.rs b/cli/src/commands/extensions/extension.rs index ee5684ec6..159bdeeb8 100644 --- a/cli/src/commands/extensions/extension.rs +++ b/cli/src/commands/extensions/extension.rs @@ -1,9 +1,9 @@ use std::convert::TryFrom; use std::fs::{self, File, Permissions}; use std::io::Read; -use std::path::PathBuf; #[cfg(unix)] use std::os::unix::prelude::PermissionsExt; +use std::path::PathBuf; use anyhow::{anyhow, Result}; use lazy_static::lazy_static; diff --git a/cli/tests/extensions.rs b/cli/tests/extensions.rs index 3dc24f422..06abc2b4b 100644 --- a/cli/tests/extensions.rs +++ b/cli/tests/extensions.rs @@ -2,8 +2,8 @@ use std::convert::TryFrom; use std::env; -use std::fs; use std::ffi::OsStr; +use std::fs; use std::path::{Path, PathBuf}; use assert_cmd::Command; @@ -82,7 +82,10 @@ fn successful_installation_prints_message() { .success(); let output = std::str::from_utf8(&cmd.get_output().stdout).unwrap(); - assert!(output.lines().find(|m| m.contains("Extension sample-extension installed successfully")).is_some()); + assert!(output + .lines() + .find(|m| m.contains("Extension sample-extension installed successfully")) + .is_some()); } // When a user attempts to install an invalid extension, it should fail and @@ -310,7 +313,9 @@ impl AsRef for TmpDir { impl Drop for TmpDir { fn drop(&mut self) { - // fs::remove_dir_all(&self.0).unwrap(); + if env::var_os("PHYLUM_TEST_SKIP_CLEANUP").is_none() { + fs::remove_dir_all(&self.0).unwrap(); + } } } From df7fac2b7778a0b8ec22cb11e169ec1e88df456a Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Thu, 19 May 2022 15:10:01 +0200 Subject: [PATCH 16/52] Anyhow error simplification --- cli/src/commands/extensions/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/src/commands/extensions/mod.rs b/cli/src/commands/extensions/mod.rs index 7618cd4e6..00fd2615d 100644 --- a/cli/src/commands/extensions/mod.rs +++ b/cli/src/commands/extensions/mod.rs @@ -118,7 +118,7 @@ fn installed_extensions() -> Result> { Ok(std::fs::read_dir(extensions_path)? .filter_map(|dir_entry| { match dir_entry - .map_err(|e| anyhow!("{}", e)) + .map_err(|e| e.into()) .and_then(|dir_entry| Extension::try_from(dir_entry.path())) { Ok(ext) => Some(ext), From 3ad428be8a81e9ca2137ae73f184acfa2d46aa78 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Thu, 19 May 2022 15:51:49 +0200 Subject: [PATCH 17/52] Lints and cargo alias --- .cargo/config | 1 + cli/src/commands/extensions/mod.rs | 2 +- cli/tests/extensions.rs | 14 ++++++-------- 3 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.cargo/config b/.cargo/config index 35049cbcb..81baff1a3 100644 --- a/.cargo/config +++ b/.cargo/config @@ -1,2 +1,3 @@ [alias] xtask = "run --package xtask --" +lints = "clippy --locked --all-features --all-targets -- -D warnings" diff --git a/cli/src/commands/extensions/mod.rs b/cli/src/commands/extensions/mod.rs index 00fd2615d..9cf17c69b 100644 --- a/cli/src/commands/extensions/mod.rs +++ b/cli/src/commands/extensions/mod.rs @@ -5,7 +5,7 @@ use std::{collections::HashSet, convert::TryFrom, path::PathBuf}; use crate::commands::{CommandResult, CommandValue, ExitCode}; pub use extension::*; -use anyhow::{anyhow, Result}; +use anyhow::Result; use clap::{arg, ArgMatches, Command, ValueHint}; use log::{error, warn}; diff --git a/cli/tests/extensions.rs b/cli/tests/extensions.rs index 06abc2b4b..210c5e605 100644 --- a/cli/tests/extensions.rs +++ b/cli/tests/extensions.rs @@ -84,8 +84,7 @@ fn successful_installation_prints_message() { let output = std::str::from_utf8(&cmd.get_output().stdout).unwrap(); assert!(output .lines() - .find(|m| m.contains("Extension sample-extension installed successfully")) - .is_some()); + .any(|m| m.contains("Extension sample-extension installed successfully"))); } // When a user attempts to install an invalid extension, it should fail and @@ -97,7 +96,7 @@ fn unsuccessful_installation_prints_failure_message() { fn stderr_match_regex(cmd: assert_cmd::assert::Assert, pattern: &str) -> bool { let output = std::str::from_utf8(&cmd.get_output().stderr).unwrap(); - output.lines().find(|m| m.contains(pattern)).is_some() + output.lines().any(|m| m.contains(pattern)) } // Install the extension. Should succeed. @@ -165,8 +164,7 @@ fn extension_is_uninstalled_correctly() { assert!( walkdir::WalkDir::new(&extension_path) .into_iter() - .collect::>() - .len() + .count() > 1 ); @@ -200,7 +198,7 @@ fn extension_list_should_emit_output() { let output = std::str::from_utf8(&cmd.get_output().stdout).unwrap(); let re = Regex::new(r#"No extension"#).unwrap(); - assert!(output.lines().find(|m| re.is_match(m)).is_some()); + assert!(output.lines().any(|m| re.is_match(m))); // Install one extension Command::cargo_bin("phylum") @@ -222,7 +220,7 @@ fn extension_list_should_emit_output() { let output = std::str::from_utf8(&cmd.get_output().stdout).unwrap(); let re = Regex::new(r#"^sample-extension\s+This extension does a thing"#).unwrap(); - assert!(output.lines().find(|m| re.is_match(m)).is_some()); + assert!(output.lines().any(|m| re.is_match(m))); } //////////////////////////////////////////////////////////////////////////////// @@ -307,7 +305,7 @@ impl AsRef for TmpDir { impl AsRef for TmpDir { fn as_ref(&self) -> &OsStr { - &self.0.as_os_str() + self.0.as_os_str() } } From 2eb4f4c37fb011d17dd5b49e7d565c42f05765a8 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Thu, 19 May 2022 17:24:44 +0200 Subject: [PATCH 18/52] Phylum Analyze extension API draft --- cli/src/commands/extensions/api.rs | 39 ++++++++++++++++++++++++++++++ cli/src/commands/extensions/mod.rs | 1 + 2 files changed, 40 insertions(+) create mode 100644 cli/src/commands/extensions/api.rs diff --git a/cli/src/commands/extensions/api.rs b/cli/src/commands/extensions/api.rs new file mode 100644 index 000000000..88f3f866e --- /dev/null +++ b/cli/src/commands/extensions/api.rs @@ -0,0 +1,39 @@ +use std::path::Path; + +use crate::api::PhylumApi; +use crate::commands::parse::get_packages_from_lockfile; +use crate::config::get_current_project; + +use anyhow::{anyhow, Context, Result}; + +pub(super) async fn phylum_analyze(api: &mut PhylumApi, lockfile: &str, project: Option<&str>, group: Option<&str>) -> Result<()> { + let (packages, request_type) = get_packages_from_lockfile(Path::new(lockfile)) + .context("Unable to locate any valid package in package lockfile")?; + + let (project, group) = match (project, group) { + (Some(project), group) => (api.get_project_id(project, group).await?, None), + (None, _) => if let Some(p) = get_current_project() { + (p.id, p.group_name) + } else { + return Err(anyhow!("Failed to find a valid project configuration")) + } + }; + + let job_id = api + .submit_request( + &request_type, + &packages, + false, + project, + None, + group.map(String::from), + ) + .await?; + + Ok(()) +} +// pub(super) fn phylum_auth_status() -> Result; +// pub(super) fn phylum_auth_token() -> Result; +// pub(super) fn phylum_history(job_id: &str) -> Result<()>; +// pub(super) fn phylum_package(name: &str, version: &str, t: Option) -> Result; +// pub(super) fn phylum_parse(lockfile: T, t: Option) -> Vec; diff --git a/cli/src/commands/extensions/mod.rs b/cli/src/commands/extensions/mod.rs index 9cf17c69b..3d0d6deba 100644 --- a/cli/src/commands/extensions/mod.rs +++ b/cli/src/commands/extensions/mod.rs @@ -1,4 +1,5 @@ pub mod extension; +mod api; use std::{collections::HashSet, convert::TryFrom, path::PathBuf}; From 29703e9d1347703f304d77b5ff09acaf32110c70 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Fri, 20 May 2022 15:46:53 +0200 Subject: [PATCH 19/52] Extensions API functions --- cli/src/commands/extensions/api.rs | 124 +++++++++++++++++++++++++---- cli/src/commands/parse.rs | 5 +- 2 files changed, 111 insertions(+), 18 deletions(-) diff --git a/cli/src/commands/extensions/api.rs b/cli/src/commands/extensions/api.rs index 88f3f866e..5f54db8f8 100644 --- a/cli/src/commands/extensions/api.rs +++ b/cli/src/commands/extensions/api.rs @@ -1,21 +1,37 @@ use std::path::Path; +use std::str::FromStr; -use crate::api::PhylumApi; -use crate::commands::parse::get_packages_from_lockfile; -use crate::config::get_current_project; +use crate::commands::parse::{get_packages_from_lockfile, LOCKFILE_PARSERS}; +use crate::config::{get_current_project, Config}; +use crate::{api::PhylumApi, auth::UserInfo}; -use anyhow::{anyhow, Context, Result}; +use anyhow::{anyhow, Context, Error, Result}; +use phylum_types::types::auth::{AccessToken, RefreshToken}; +use phylum_types::types::common::{JobId, ProjectId}; +use phylum_types::types::job::JobStatusResponse; +use phylum_types::types::package::{ + Package, PackageDescriptor, PackageStatusExtended, PackageType, +}; +use phylum_types::types::project::ProjectDetailsResponse; -pub(super) async fn phylum_analyze(api: &mut PhylumApi, lockfile: &str, project: Option<&str>, group: Option<&str>) -> Result<()> { +#[allow(unused)] +pub(super) async fn phylum_analyze( + api: &mut PhylumApi, + lockfile: &str, + project: Option<&str>, + group: Option<&str>, +) -> Result { let (packages, request_type) = get_packages_from_lockfile(Path::new(lockfile)) .context("Unable to locate any valid package in package lockfile")?; let (project, group) = match (project, group) { (Some(project), group) => (api.get_project_id(project, group).await?, None), - (None, _) => if let Some(p) = get_current_project() { - (p.id, p.group_name) - } else { - return Err(anyhow!("Failed to find a valid project configuration")) + (None, _) => { + if let Some(p) = get_current_project() { + (p.id, p.group_name) + } else { + return Err(anyhow!("Failed to find a valid project configuration")); + } } }; @@ -30,10 +46,88 @@ pub(super) async fn phylum_analyze(api: &mut PhylumApi, lockfile: &str, project: ) .await?; - Ok(()) + Ok(job_id) +} + +#[allow(unused)] +pub(super) async fn phylum_auth_status(api: &mut PhylumApi, config: &Config) -> Result { + api.user_info(&config.auth_info).await.map_err(Error::from) +} + +#[allow(unused)] +pub(super) async fn phylum_auth_token_bearer(config: &Config) -> Result { + let refresh_token = phylum_auth_token(config)?; + let access_token = crate::auth::handle_refresh_tokens(&config.auth_info, &refresh_token) + .await? + .access_token; + Ok(access_token) +} + +#[allow(unused)] +pub(super) fn phylum_auth_token(config: &Config) -> Result { + config + .auth_info + .offline_access + .clone() + .ok_or_else(|| anyhow!("User is not currently authenticated")) +} + +#[allow(unused)] +pub(super) async fn phylum_history_job( + api: &mut PhylumApi, + job_id: Option<&str>, +) -> Result> { + let job_id = job_id + .map(|job_id| JobId::from_str(job_id).ok()) + .unwrap_or_else(|| get_current_project().map(|p| p.id)) + .ok_or_else(|| anyhow!("Failed to find a valid project configuration"))?; + api.get_job_status_ext(&job_id).await.map_err(Error::from) +} + +#[allow(unused)] +pub(super) async fn phylum_history_project( + api: &mut PhylumApi, + project_name: Option<&str>, +) -> Result { + let project_name = project_name + .map(String::from) + .map(Result::Ok) + .unwrap_or_else(|| { + get_current_project() + .map(|p| p.name) + .ok_or_else(|| anyhow!("Failed to find a valid project configuration")) + })?; + api.get_project_details(&project_name) + .await + .map_err(Error::from) +} + +#[allow(unused)] +pub(super) async fn phylum_package( + api: &mut PhylumApi, + name: &str, + version: &str, + package_type: &str, +) -> Result { + let package_type = PackageType::from_str(package_type) + .map_err(|e| anyhow!("Unrecognized package type `{package_type}`: {e:?}"))?; + api.get_package_details(&PackageDescriptor { + name: name.to_string(), + version: version.to_string(), + package_type, + }) + .await + .map_err(Error::from) +} + +#[allow(unused)] +pub(super) fn phylum_parse(lockfile: &str, lockfile_type: &str) -> Result> { + let parser = LOCKFILE_PARSERS + .iter() + .find_map(|(name, parser)| (*name == lockfile_type).then(|| parser)) + .ok_or_else(|| anyhow!("Unrecognized lockfile type: `{lockfile_type}`"))?; + + let (pkgs, _) = parser(Path::new(lockfile))?; + + Ok(pkgs) } -// pub(super) fn phylum_auth_status() -> Result; -// pub(super) fn phylum_auth_token() -> Result; -// pub(super) fn phylum_history(job_id: &str) -> Result<()>; -// pub(super) fn phylum_package(name: &str, version: &str, t: Option) -> Result; -// pub(super) fn phylum_parse(lockfile: T, t: Option) -> Vec; diff --git a/cli/src/commands/parse.rs b/cli/src/commands/parse.rs index 8dc70a806..094bf623f 100644 --- a/cli/src/commands/parse.rs +++ b/cli/src/commands/parse.rs @@ -11,7 +11,7 @@ use crate::lockfiles::*; type ParserResult = Result<(Vec, PackageType)>; -const LOCKFILE_PARSERS: &[(&str, &dyn Fn(&Path) -> ParserResult)] = &[ +pub(crate) const LOCKFILE_PARSERS: &[(&str, &dyn Fn(&Path) -> ParserResult)] = &[ ("yarn", &parse::), ("npm", &parse::), ("gem", &parse::), @@ -35,8 +35,7 @@ pub fn handle_parse(matches: &clap::ArgMatches) -> CommandResult { let parser = LOCKFILE_PARSERS .iter() - .filter_map(|(name, parser)| (*name == lockfile_type).then(|| parser)) - .next() + .find_map(|(name, parser)| (*name == lockfile_type).then(|| parser)) .unwrap(); let (pkgs, _) = parser(Path::new(lockfile))?; From bf2b1ddd2aa72a489dc87587a03456f17cd7bf9d Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Fri, 20 May 2022 17:15:44 +0200 Subject: [PATCH 20/52] Cargo fmt --- cli/tests/extensions.rs | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/cli/tests/extensions.rs b/cli/tests/extensions.rs index 210c5e605..5a5697eab 100644 --- a/cli/tests/extensions.rs +++ b/cli/tests/extensions.rs @@ -161,12 +161,7 @@ fn extension_is_uninstalled_correctly() { .join("extensions") .join("sample-extension"); - assert!( - walkdir::WalkDir::new(&extension_path) - .into_iter() - .count() - > 1 - ); + assert!(walkdir::WalkDir::new(&extension_path).into_iter().count() > 1); Command::cargo_bin("phylum") .unwrap() From 6e0f5cbfcad98886ba75e30e3489d309bd07cd77 Mon Sep 17 00:00:00 2001 From: andreaphylum <68552656+andreaphylum@users.noreply.github.com> Date: Fri, 20 May 2022 19:59:45 +0200 Subject: [PATCH 21/52] Apply suggestions from code review Co-authored-by: Kyle Willmon --- cli/src/commands/extensions/extension.rs | 3 +-- cli/src/commands/extensions/mod.rs | 23 +++++++++++++---------- cli/src/config.rs | 3 +-- 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/cli/src/commands/extensions/extension.rs b/cli/src/commands/extensions/extension.rs index 159bdeeb8..529d9868f 100644 --- a/cli/src/commands/extensions/extension.rs +++ b/cli/src/commands/extensions/extension.rs @@ -121,8 +121,7 @@ impl TryFrom for Extension { )); } - let mut buf = Vec::new(); - File::open(manifest_path)?.read_to_end(&mut buf)?; + let buf = fs::read(manifest_path)?; let manifest: ExtensionManifest = toml::from_slice(&buf)?; let entry_point_path = path.join(&manifest.entry_point); diff --git a/cli/src/commands/extensions/mod.rs b/cli/src/commands/extensions/mod.rs index 9cf17c69b..7dd601ef7 100644 --- a/cli/src/commands/extensions/mod.rs +++ b/cli/src/commands/extensions/mod.rs @@ -61,13 +61,11 @@ pub fn extensions_subcommands(command: Command<'_>) -> Command<'_> { /// Entry point for the `extensions` subcommand. pub async fn handle_extensions(matches: &ArgMatches) -> CommandResult { - if let Some(matches) = matches.subcommand_matches("add") { - handle_add_extension(matches.value_of_t("PATH")?).await - } else if let Some(matches) = matches.subcommand_matches("remove") { - handle_remove_extension(matches.value_of("NAME").unwrap()).await - } else { - // also covers the `list` case - handle_list_extensions().await + match matches.subcommand() { + Some(("add", matches)) => handle_add_extension(matches.value_of_t("PATH")?).await, + Some(("remove", matches)) => handle_remove_extension(matches.value_of("NAME").unwrap()).await, + Some(("list", _)) | None => handle_list_extensions().await, + _ => unreachable!(), } } @@ -111,11 +109,16 @@ async fn handle_list_extensions() -> CommandResult { fn installed_extensions() -> Result> { let extensions_path = extensions_path()?; - if !extensions_path.exists() { - return Ok(Vec::new()); + let dir_entry = match fs::read_dir(extensions_path) { + Ok(d) => d, + Err(e) => if e.kind() == ErrorKind::NotFound { + return Ok(Vec::new()); + } else { + return Err(e.into()); + } } - Ok(std::fs::read_dir(extensions_path)? + Ok(dir_entry .filter_map(|dir_entry| { match dir_entry .map_err(|e| e.into()) diff --git a/cli/src/config.rs b/cli/src/config.rs index 9b90a7231..174043e6c 100644 --- a/cli/src/config.rs +++ b/cli/src/config.rs @@ -197,8 +197,7 @@ pub fn get_home_settings_path() -> Result { /// Resolve XDG data directory. pub fn data_dir() -> Result { - if let Some(data_dir) = env::var("XDG_DATA_HOME") - .ok() + if let Some(data_dir) = env::var_os("XDG_DATA_HOME") .filter(|s| !s.is_empty()) .map(PathBuf::from) { From ba7ac0377e9a1d7c849e873fe2b1e3e0f8b6e09f Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Fri, 20 May 2022 20:12:01 +0200 Subject: [PATCH 22/52] More code review fixes --- cli/src/commands/extensions/extension.rs | 3 +-- cli/src/commands/extensions/mod.rs | 20 ++++++++++++-------- docs/phylum_extension_add.md | 1 + 3 files changed, 14 insertions(+), 10 deletions(-) diff --git a/cli/src/commands/extensions/extension.rs b/cli/src/commands/extensions/extension.rs index 529d9868f..7ec69fd7f 100644 --- a/cli/src/commands/extensions/extension.rs +++ b/cli/src/commands/extensions/extension.rs @@ -1,6 +1,5 @@ use std::convert::TryFrom; -use std::fs::{self, File, Permissions}; -use std::io::Read; +use std::fs::{self, Permissions}; #[cfg(unix)] use std::os::unix::prelude::PermissionsExt; use std::path::PathBuf; diff --git a/cli/src/commands/extensions/mod.rs b/cli/src/commands/extensions/mod.rs index 7dd601ef7..c01d64785 100644 --- a/cli/src/commands/extensions/mod.rs +++ b/cli/src/commands/extensions/mod.rs @@ -1,6 +1,6 @@ -pub mod extension; +mod extension; -use std::{collections::HashSet, convert::TryFrom, path::PathBuf}; +use std::{collections::HashSet, convert::TryFrom, fs, io::ErrorKind, path::PathBuf}; use crate::commands::{CommandResult, CommandValue, ExitCode}; pub use extension::*; @@ -63,7 +63,9 @@ pub fn extensions_subcommands(command: Command<'_>) -> Command<'_> { pub async fn handle_extensions(matches: &ArgMatches) -> CommandResult { match matches.subcommand() { Some(("add", matches)) => handle_add_extension(matches.value_of_t("PATH")?).await, - Some(("remove", matches)) => handle_remove_extension(matches.value_of("NAME").unwrap()).await, + Some(("remove", matches)) => { + handle_remove_extension(matches.value_of("NAME").unwrap()).await + } Some(("list", _)) | None => handle_list_extensions().await, _ => unreachable!(), } @@ -111,12 +113,14 @@ fn installed_extensions() -> Result> { let dir_entry = match fs::read_dir(extensions_path) { Ok(d) => d, - Err(e) => if e.kind() == ErrorKind::NotFound { - return Ok(Vec::new()); - } else { - return Err(e.into()); + Err(e) => { + if e.kind() == ErrorKind::NotFound { + return Ok(Vec::new()); + } else { + return Err(e.into()); + } } - } + }; Ok(dir_entry .filter_map(|dir_entry| { diff --git a/docs/phylum_extension_add.md b/docs/phylum_extension_add.md index 59933c0ef..ed9b413ea 100644 --- a/docs/phylum_extension_add.md +++ b/docs/phylum_extension_add.md @@ -10,6 +10,7 @@ phylum extension add path/to/extension ``` The extension will be installed under `$XDG_DATA_HOME/phylum/extensions/`. +If `$XDG_DATA_HOME` is not set, it will default to `$HOME/.local/share/phylum/extensions/`. Once installed, the extension will be accessible via the Phylum CLI: ```sh From 6dc926b83a1cdad21ac43bd24874e238badad006 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Mon, 23 May 2022 15:07:09 +0200 Subject: [PATCH 23/52] Fix typo --- cli/Cargo.toml | 1 - 1 file changed, 1 deletion(-) diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 2ffa7bafd..8b6ec2038 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -52,7 +52,6 @@ toml = "0.5.8" unicode-width = "0.1.9" url = { version = "2", features = ["serde"] } zip = "0.6.2" -unicode-width = "0.1.9" walkdir = "2.3.2" regex = "1.5.5" From 211b99e3aab541d579abe931b021369202fad7fc Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Mon, 23 May 2022 18:47:23 +0200 Subject: [PATCH 24/52] Injected dependencies and `deno_core::OpState` support --- Cargo.lock | 986 ++++++++++++++++++++++++++++- cli/Cargo.toml | 3 + cli/src/commands/extensions/api.rs | 56 +- 3 files changed, 1024 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 50872bf8f..4032023e1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,16 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "Inflector" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fe438c63458706e03479442743baae6c88256498e6431708f6dfc520a26515d3" +dependencies = [ + "lazy_static", + "regex", +] + [[package]] name = "adler" version = "1.0.2" @@ -20,6 +30,17 @@ dependencies = [ "opaque-debug", ] +[[package]] +name = "ahash" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fcb51a0695d8f838b1ee009b3fbf66bda078cd64590202a864a8f3e8c4315c47" +dependencies = [ + "getrandom 0.2.6", + "once_cell", + "version_check", +] + [[package]] name = "aho-corasick" version = "0.7.18" @@ -80,6 +101,20 @@ dependencies = [ "wait-timeout", ] +[[package]] +name = "ast_node" +version = "0.7.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bc4c00309ed1c8104732df4a5fa9acc3b796b6f8531dfbd5ce0078c86f997244" +dependencies = [ + "darling", + "pmutil", + "proc-macro2", + "quote", + "swc_macros_common", + "syn", +] + [[package]] name = "async-channel" version = "1.6.1" @@ -119,6 +154,12 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" +[[package]] +name = "base64" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" + [[package]] name = "base64" version = "0.13.0" @@ -131,6 +172,15 @@ version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a32fd6af2b5827bce66c29053ba0e7c42b9dcab01835835058558c10851a46b" +[[package]] +name = "better_scoped_tls" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b73e8ecdec39e98aa3b19e8cd0b8ed8f77ccb86a6b0b2dc7cd86d105438a2123" +dependencies = [ + "scoped-tls", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -268,7 +318,7 @@ dependencies = [ "bitflags", "clap_lex", "indexmap", - "strsim", + "strsim 0.10.0", "termcolor", "textwrap", "yaml-rust", @@ -322,6 +372,12 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" +[[package]] +name = "convert_case" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e" + [[package]] name = "cpufeatures" version = "0.2.2" @@ -382,6 +438,61 @@ dependencies = [ "memchr", ] +[[package]] +name = "darling" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d706e75d87e35569db781a9b5e2416cff1236a47ed380831f959382ccd5f858" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c960ae2da4de88a91b2d920c2a7233b400bc33cb28453a2987822d8392519b" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.9.3", + "syn", +] + +[[package]] +name = "darling_macro" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9b5a2f4ac4969822c62224815d069952656cadc7084fdca9751e6d959189b72" +dependencies = [ + "darling_core", + "quote", + "syn", +] + +[[package]] +name = "dashmap" +version = "5.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "391b56fbd302e585b7a9494fb70e40949567b1cf9003a8e4a6041a1687c26573" +dependencies = [ + "cfg-if", + "hashbrown 0.12.1", + "lock_api", +] + +[[package]] +name = "data-url" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a30bfce702bcfa94e906ef82421f2c0e61c076ad76030c16ee5d2e9a32fe193" +dependencies = [ + "matches", +] + [[package]] name = "deadpool" version = "0.9.4" @@ -401,6 +512,80 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eaa37046cc0f6c3cc6090fbdbf73ef0b8ef4cfcc37f6befc0020f63e8cf121e1" +[[package]] +name = "debug_unreachable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a032eac705ca39214d169f83e3d3da290af06d8d1d344d1baad2fd002dca4b3" +dependencies = [ + "unreachable", +] + +[[package]] +name = "deno_ast" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0691d08faa4f1fd9898ecfbe82c32623f89969e92e7b97b5119a18399609d25" +dependencies = [ + "anyhow", + "base64 0.13.0", + "data-url", + "serde", + "swc_atoms", + "swc_common", + "swc_ecmascript", + "text_lines", + "url", +] + +[[package]] +name = "deno_core" +version = "0.135.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cd837520179a6f8063fe542b98dacec14b44ce647990be476b6eca8e6125e5" +dependencies = [ + "anyhow", + "deno_ops", + "futures", + "indexmap", + "libc", + "log", + "once_cell", + "parking_lot", + "pin-project", + "serde", + "serde_json", + "serde_v8", + "sourcemap", + "url", + "v8", +] + +[[package]] +name = "deno_ops" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6ab6a5a7c3d5b9fbd43064996bbe61799db5e0bfb0f46672b2f85c0192d15a9" +dependencies = [ + "proc-macro-crate", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive_more" +version = "0.99.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb810d30a7c1953f91334de7244731fc3f3c10d7fe163338a35b9f640960321" +dependencies = [ + "convert_case", + "proc-macro2", + "quote", + "rustc_version 0.4.0", + "syn", +] + [[package]] name = "dialoguer" version = "0.10.1" @@ -488,6 +673,18 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "enum_kind" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "78b940da354ae81ef0926c5eaa428207b8f4f091d3956c891dfbd124162bed99" +dependencies = [ + "pmutil", + "proc-macro2", + "swc_macros_common", + "syn", +] + [[package]] name = "env_logger" version = "0.9.0" @@ -544,12 +741,34 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "from_variant" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0951635027ca477be98f8774abd6f0345233439d63f307e47101acb40c7cc63d" +dependencies = [ + "pmutil", + "proc-macro2", + "swc_macros_common", + "syn", +] + [[package]] name = "fs_extra" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2022715d62ab30faffd124d40b76f4134a550a87792276512b18d63272333394" +[[package]] +name = "fslock" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57eafdd0c16f57161105ae1b98a1238f97645f2f588438b2949c99a2af9616bf" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "futures" version = "0.3.21" @@ -739,6 +958,12 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e" +[[package]] +name = "hashbrown" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "db0d4cf898abf0081f964436dc980e96670a0f36863e4b83aaacdb65c9d7ccc3" + [[package]] name = "heck" version = "0.4.0" @@ -802,7 +1027,7 @@ checksum = "6e9b187a72d63adbfba487f48095306ac823049cb504ee195541e91c7775f5ad" dependencies = [ "anyhow", "async-channel", - "base64", + "base64 0.13.0", "futures-lite", "http", "infer", @@ -870,6 +1095,12 @@ dependencies = [ "tokio-rustls", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "0.2.3" @@ -881,6 +1112,12 @@ dependencies = [ "unicode-normalization", ] +[[package]] +name = "if_chain" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb56e1aa765b4b4f3aadfab769793b7087bb03a4ea4920644a6d238e2df5b9ed" + [[package]] name = "indexmap" version = "1.8.1" @@ -888,7 +1125,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee" dependencies = [ "autocfg", - "hashbrown", + "hashbrown 0.11.2", ] [[package]] @@ -912,6 +1149,19 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "879d54834c8c76457ef4293a689b2a8c59b076067ad77b15efafbb05f92a592b" +[[package]] +name = "is-macro" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94b2c46692aee0d1b3aad44e781ac0f0e7db42ef27adaa0a877b627040019813" +dependencies = [ + "Inflector", + "pmutil", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "itertools" version = "0.10.3" @@ -957,6 +1207,79 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" +[[package]] +name = "lexical" +version = "6.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c7aefb36fd43fef7003334742cbf77b243fcd36418a1d1bdd480d613a67968f6" +dependencies = [ + "lexical-core", +] + +[[package]] +name = "lexical-core" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2cde5de06e8d4c2faabc400238f9ae1c74d5412d03a7bd067645ccbc47070e46" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683b3a5ebd0130b8fb52ba0bdc718cc56815b6a097e28ae5a6997d0ad17dc05f" +dependencies = [ + "lexical-parse-integer", + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-parse-integer" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "125e1f93e5003d4bd89758c2ca2771bfae13632df633cde581efe07c87d354e5" +dependencies = [ + "lexical-util", + "static_assertions", +] + +[[package]] +name = "lexical-util" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5255b9ff16ff898710eb9eb63cb39248ea8a5bb036bea8085b1a767ff6c4e3fc" +dependencies = [ + "static_assertions", +] + +[[package]] +name = "lexical-write-float" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "accabaa1c4581f05a3923d1b4cfd124c329352288b7b9da09e766b0668116862" +dependencies = [ + "lexical-util", + "lexical-write-integer", + "static_assertions", +] + +[[package]] +name = "lexical-write-integer" +version = "0.8.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1b6f3d1f4422866b68192d62f77bc5c700bee84f3069f2469d7bc8c77852446" +dependencies = [ + "lexical-util", + "static_assertions", +] + [[package]] name = "libc" version = "0.2.125" @@ -1045,6 +1368,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "new_debug_unreachable" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4a24736216ec316047a1fc4252e27dabb04218aa4a3f37c6e7ddbf1f9782b54" + [[package]] name = "nom" version = "7.1.1" @@ -1055,6 +1384,18 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "num-bigint" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f93ab6289c7b344a8a9f60f88d80aa20032336fe78da341afc91c8a2341fc75f" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", + "serde", +] + [[package]] name = "num-integer" version = "0.1.45" @@ -1095,9 +1436,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.10.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9" +checksum = "7709cef83f0c1f58f666e746a08b21e0085f7440fa6a29cc194d68aac97a4225" [[package]] name = "opaque-debug" @@ -1185,6 +1526,50 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" +[[package]] +name = "phf" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259" +dependencies = [ + "phf_macros", + "phf_shared", + "proc-macro-hack", +] + +[[package]] +name = "phf_generator" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6" +dependencies = [ + "phf_shared", + "rand 0.8.5", +] + +[[package]] +name = "phf_macros" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "58fdf3184dd560f160dd73922bea2d5cd6e8f064bf4b13110abd81b03697b4e0" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro-hack", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "phf_shared" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" +dependencies = [ + "siphasher", +] + [[package]] name = "phylum-cli" version = "3.4.0" @@ -1192,11 +1577,13 @@ dependencies = [ "ansi_term", "anyhow", "assert_cmd", - "base64", + "base64 0.13.0", "bytes", "chrono", "cidr", "clap", + "deno_ast", + "deno_core", "dialoguer", "env_logger", "futures", @@ -1208,6 +1595,7 @@ dependencies = [ "maplit", "minisign-verify", "nom", + "once_cell", "open", "phylum_types", "prettytable-rs", @@ -1247,10 +1635,30 @@ dependencies = [ ] [[package]] -name = "pin-project-lite" -version = "0.2.9" +name = "pin-project" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" +checksum = "58ad3879ad3baf4e44784bc6a718a8698867bb991f8ce24d1bcbe2cfb4c3a75e" +dependencies = [ + "pin-project-internal", +] + +[[package]] +name = "pin-project-internal" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "744b6f092ba29c3650faf274db506afd39944f48420f6c86b17cfe0ee1cb36bb" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0a7ae3ac2f1173085d398531c705756c94a4c56843785df85a60c1a0afac116" [[package]] name = "pin-utils" @@ -1264,12 +1672,29 @@ version = "0.3.25" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae" +[[package]] +name = "pmutil" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3894e5d549cccbe44afecf72922f277f603cd4bb0219c8342631ef18fffbe004" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "ppv-lite86" version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb9f9e6e233e5c4a35559a617bf40a4ec447db2e84c20b55a6f83167b7e57872" +[[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + [[package]] name = "predicates" version = "2.1.1" @@ -1311,6 +1736,16 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "proc-macro-crate" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e17d47ce914bf4de440332250b0edd23ce48c005f59fab39d3335866b114f11a" +dependencies = [ + "thiserror", + "toml", +] + [[package]] name = "proc-macro-hack" version = "0.5.19" @@ -1481,7 +1916,7 @@ version = "0.11.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46a1f7aa4f35e5e8b4160449f51afc758f0ce6454315a9fa7d0d113e958c41eb" dependencies = [ - "base64", + "base64 0.13.0", "bytes", "encoding_rs", "futures-core", @@ -1553,12 +1988,36 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4b18820d944b33caa75a71378964ac46f58517c92b6ae5f762636247c09e78fb" dependencies = [ - "base64", + "base64 0.13.0", "blake2b_simd", "constant_time_eq", "crossbeam-utils", ] +[[package]] +name = "rustc-hash" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver 0.9.0", +] + +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver 1.0.9", +] + [[package]] name = "rustls" version = "0.20.5" @@ -1577,7 +2036,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ee86d63972a7c661d1536fefe8c3c8407321c3df668891286de28abcd087360" dependencies = [ - "base64", + "base64 0.13.0", ] [[package]] @@ -1601,6 +2060,12 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scoped-tls" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea6a9290e3c9cf0f18145ef7ffa62d68ee0bf5fcd651017e586dc7fd5da448c2" + [[package]] name = "scopeguard" version = "1.1.0" @@ -1617,6 +2082,27 @@ dependencies = [ "untrusted", ] +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cb243bdfdb5936c8dc3c45762a19d12ab4550cdc753bc247637d4ec35a040fd" + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + [[package]] name = "serde" version = "1.0.137" @@ -1655,6 +2141,7 @@ version = "1.0.81" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c" dependencies = [ + "indexmap", "itoa 1.0.2", "ryu", "serde", @@ -1683,6 +2170,18 @@ dependencies = [ "serde", ] +[[package]] +name = "serde_v8" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7797d56c9575ced9175e22366e5bd4cc8f3d571cd8c75be510f410ab97a54f6" +dependencies = [ + "bytes", + "derive_more", + "serde", + "v8", +] + [[package]] name = "serde_yaml" version = "0.8.24" @@ -1695,6 +2194,17 @@ dependencies = [ "yaml-rust", ] +[[package]] +name = "sha-1" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "028f48d513f9678cda28f6e4064755b3fbb2af6acd672f2c209b62323f7aea0f" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sha1" version = "0.10.1" @@ -1746,6 +2256,12 @@ dependencies = [ "time 0.3.9", ] +[[package]] +name = "siphasher" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7bd3e3206899af3f8b12af284fafc038cc1dc2b41d1b89dd17297221c5d225de" + [[package]] name = "slab" version = "0.4.6" @@ -1774,6 +2290,22 @@ dependencies = [ "winapi", ] +[[package]] +name = "sourcemap" +version = "6.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6e031f2463ecbdd5f34c950f89f5c1e1032f22c0f8e3dc4bdb2e8b6658cf61eb" +dependencies = [ + "base64 0.11.0", + "if_chain", + "lazy_static", + "regex", + "rustc_version 0.2.3", + "serde", + "serde_json", + "url", +] + [[package]] name = "spin" version = "0.5.2" @@ -1791,6 +2323,57 @@ dependencies = [ "strum", ] +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "string_cache" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "213494b7a2b503146286049378ce02b482200519accc31872ee8be91fa820a08" +dependencies = [ + "new_debug_unreachable", + "once_cell", + "parking_lot", + "phf_shared", + "precomputed-hash", + "serde", +] + +[[package]] +name = "string_cache_codegen" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988" +dependencies = [ + "phf_generator", + "phf_shared", + "proc-macro2", + "quote", +] + +[[package]] +name = "string_enum" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f584cc881e9e5f1fd6bf827b0444aa94c30d8fe6378cf241071b5f5700b2871f" +dependencies = [ + "pmutil", + "proc-macro2", + "quote", + "swc_macros_common", + "syn", +] + +[[package]] +name = "strsim" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6446ced80d6c486436db5c078dde11a9f73d42b57fb273121e160b84f63d894c" + [[package]] name = "strsim" version = "0.10.0" @@ -1825,6 +2408,325 @@ version = "2.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601" +[[package]] +name = "swc_atoms" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba8735ce37e421749498e038955abc1135eec6a4af0b54a173e55d2e5542d472" +dependencies = [ + "string_cache", + "string_cache_codegen", +] + +[[package]] +name = "swc_common" +version = "0.17.25" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "766ad22c1cb8586c038ccba7371a4903a6074b53ee4ba8980a52f502413f120e" +dependencies = [ + "ahash", + "ast_node", + "better_scoped_tls", + "cfg-if", + "debug_unreachable", + "either", + "from_variant", + "num-bigint", + "once_cell", + "rustc-hash", + "serde", + "siphasher", + "sourcemap", + "string_cache", + "swc_eq_ignore_macros", + "swc_visit", + "tracing", + "unicode-width", + "url", +] + +[[package]] +name = "swc_ecma_ast" +version = "0.75.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72961898fbe56591997e667a1ec6a268383582810351c279a15ec710b6177d33" +dependencies = [ + "is-macro", + "num-bigint", + "serde", + "string_enum", + "swc_atoms", + "swc_common", + "unicode-id", +] + +[[package]] +name = "swc_ecma_codegen" +version = "0.103.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "99ca430d8ea2c8791d1341c4035431c90b87330e39479b4a6dabb4fded124e30" +dependencies = [ + "bitflags", + "memchr", + "num-bigint", + "once_cell", + "rustc-hash", + "sourcemap", + "swc_atoms", + "swc_common", + "swc_ecma_ast", + "swc_ecma_codegen_macros", + "tracing", +] + +[[package]] +name = "swc_ecma_codegen_macros" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "59949619b2ef45eedb6c399d05f2c3c7bc678b5074b3103bb670f9e05bb99042" +dependencies = [ + "pmutil", + "proc-macro2", + "quote", + "swc_macros_common", + "syn", +] + +[[package]] +name = "swc_ecma_parser" +version = "0.100.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "890d967031e3e7330cd7892f27d826b7b4f37c7caa19db85c78a0862e1fe3974" +dependencies = [ + "either", + "enum_kind", + "lexical", + "num-bigint", + "serde", + "smallvec", + "swc_atoms", + "swc_common", + "swc_ecma_ast", + "tracing", + "typed-arena", + "unicode-id", +] + +[[package]] +name = "swc_ecma_transforms" +version = "0.142.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f20e5e2d8ab843fa0454e049f73f6d99c444a8c0e2320f77028361ab75e2d18e" +dependencies = [ + "swc_atoms", + "swc_common", + "swc_ecma_ast", + "swc_ecma_transforms_base", + "swc_ecma_transforms_proposal", + "swc_ecma_transforms_react", + "swc_ecma_transforms_typescript", + "swc_ecma_utils", + "swc_ecma_visit", +] + +[[package]] +name = "swc_ecma_transforms_base" +version = "0.75.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "404c6ea7ca61ceb2ce1f4ed448d1436a38c31b8c572850f04541c0229c966bbf" +dependencies = [ + "better_scoped_tls", + "once_cell", + "phf", + "serde", + "smallvec", + "swc_atoms", + "swc_common", + "swc_ecma_ast", + "swc_ecma_parser", + "swc_ecma_utils", + "swc_ecma_visit", + "tracing", +] + +[[package]] +name = "swc_ecma_transforms_classes" +version = "0.63.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "503f2f6bd0f9e6363a93406753bf64675163423774256a267c85a5d9b5b44b08" +dependencies = [ + "swc_atoms", + "swc_common", + "swc_ecma_ast", + "swc_ecma_transforms_base", + "swc_ecma_utils", + "swc_ecma_visit", +] + +[[package]] +name = "swc_ecma_transforms_macros" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "18712e4aab969c6508dff3540ade6358f1e013464aa58b3d30da2ab2d9fcbbed" +dependencies = [ + "pmutil", + "proc-macro2", + "quote", + "swc_macros_common", + "syn", +] + +[[package]] +name = "swc_ecma_transforms_proposal" +version = "0.97.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93d08411e517736b0167f3c9784fe9b98cc09308ae12e6072abd2bb2c2236da2" +dependencies = [ + "either", + "serde", + "smallvec", + "swc_atoms", + "swc_common", + "swc_ecma_ast", + "swc_ecma_transforms_base", + "swc_ecma_transforms_classes", + "swc_ecma_transforms_macros", + "swc_ecma_utils", + "swc_ecma_visit", +] + +[[package]] +name = "swc_ecma_transforms_react" +version = "0.104.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43cda44270dfcc95d61582981baddaf53d96c5233ea7384e81cd6e462816c58e" +dependencies = [ + "ahash", + "base64 0.13.0", + "dashmap", + "indexmap", + "once_cell", + "regex", + "serde", + "sha-1", + "string_enum", + "swc_atoms", + "swc_common", + "swc_ecma_ast", + "swc_ecma_parser", + "swc_ecma_transforms_base", + "swc_ecma_transforms_macros", + "swc_ecma_utils", + "swc_ecma_visit", +] + +[[package]] +name = "swc_ecma_transforms_typescript" +version = "0.107.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a09397169ed7ce0751a82cb71655f3a4a1fb00d8863aabd5cca9b46eff3dd5f2" +dependencies = [ + "serde", + "swc_atoms", + "swc_common", + "swc_ecma_ast", + "swc_ecma_transforms_base", + "swc_ecma_transforms_react", + "swc_ecma_utils", + "swc_ecma_visit", +] + +[[package]] +name = "swc_ecma_utils" +version = "0.79.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44ee8d60b9977f58214af7102dc30855a6754e742afe6d6e26e5bf13883c7b91" +dependencies = [ + "indexmap", + "once_cell", + "swc_atoms", + "swc_common", + "swc_ecma_ast", + "swc_ecma_visit", + "tracing", +] + +[[package]] +name = "swc_ecma_visit" +version = "0.61.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5ea00a52ba2b971955c62275696d5c59f3cf0cd06db74a66dec378ec9843c78" +dependencies = [ + "num-bigint", + "swc_atoms", + "swc_common", + "swc_ecma_ast", + "swc_visit", + "tracing", +] + +[[package]] +name = "swc_ecmascript" +version = "0.143.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebda93aa6422956c184a9eb5fdb0f0f0ff433169fa15e55ef445e5ad0b5e0abe" +dependencies = [ + "swc_ecma_ast", + "swc_ecma_codegen", + "swc_ecma_parser", + "swc_ecma_transforms", + "swc_ecma_utils", + "swc_ecma_visit", +] + +[[package]] +name = "swc_eq_ignore_macros" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c8f200a2eaed938e7c1a685faaa66e6d42fa9e17da5f62572d3cbc335898f5e" +dependencies = [ + "pmutil", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "swc_macros_common" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5dca3f08d02da4684c3373150f7c045128f81ea00f0c434b1b012bc65a6cce3" +dependencies = [ + "pmutil", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "swc_visit" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5c639379dd2a8a0221fa1e12fafbdd594ba53a0cace6560054da52409dfcc1a" +dependencies = [ + "either", + "swc_visit_macros", +] + +[[package]] +name = "swc_visit_macros" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3b9b72892df873972549838bf84d6c56234c7502148a7e23b5a3da6e0fedfb8" +dependencies = [ + "Inflector", + "pmutil", + "proc-macro2", + "quote", + "swc_macros_common", + "syn", +] + [[package]] name = "syn" version = "1.0.94" @@ -1886,6 +2788,15 @@ version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "507e9898683b6c43a9aa55b64259b721b52ba226e0f3779137e50ad114a4c90b" +[[package]] +name = "text_lines" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e49e3c53dd04de8b8e8390bc4fab57f6db7af7d33b086fe411803e6351c9f9f9" +dependencies = [ + "serde", +] + [[package]] name = "textwrap" version = "0.15.0" @@ -2069,6 +2980,12 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "59547bce71d9c38b83d9c0e92b6066c4253371f15005def0c30d9657f50c7642" +[[package]] +name = "typed-arena" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0685c84d5d54d1c26f7d3eb96cd41550adb97baed141a761cf335d3d33bcd0ae" + [[package]] name = "typenum" version = "1.15.0" @@ -2081,6 +2998,12 @@ version = "0.3.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "099b7128301d285f79ddd55b9a83d5e6b9e97c92e0ea0daebee7263e932de992" +[[package]] +name = "unicode-id" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69fe8d9274f490a36442acb4edfd0c4e473fdfc6a8b5cd32f28a0235761aedbe" + [[package]] name = "unicode-linebreak" version = "0.1.2" @@ -2111,6 +3034,15 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04" +[[package]] +name = "unreachable" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1f2ae5ddb18e1c92664717616dd9549dde73f539f01bd7b77c2edb2446bdff91" +dependencies = [ + "void", +] + [[package]] name = "untrusted" version = "0.7.1" @@ -2140,12 +3072,31 @@ dependencies = [ "serde", ] +[[package]] +name = "v8" +version = "0.42.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "854740dcc66681c3e10d15a1ebe1e0dfed77a7e2e58b97913dd3a111cf4cdb5f" +dependencies = [ + "bitflags", + "fslock", + "lazy_static", + "libc", + "which", +] + [[package]] name = "version_check" version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" +[[package]] +name = "void" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a02e4885ed3bc0f2de90ea6dd45ebcbb66dacffe03547fadbb0eeae2770887d" + [[package]] name = "wait-timeout" version = "0.2.0" @@ -2295,6 +3246,17 @@ dependencies = [ "webpki", ] +[[package]] +name = "which" +version = "4.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c4fb54e6113b6a8772ee41c3404fb0301ac79604489467e0a9ce1f3e97c24ae" +dependencies = [ + "either", + "lazy_static", + "libc", +] + [[package]] name = "winapi" version = "0.3.9" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 8b6ec2038..b7053e8fb 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -54,6 +54,9 @@ url = { version = "2", features = ["serde"] } zip = "0.6.2" walkdir = "2.3.2" regex = "1.5.5" +deno_core = "0.135.0" +deno_ast = { version = "0.14.0", features = ["transpiling"] } +once_cell = "1.12.0" [dev-dependencies] assert_cmd = "2.0.4" diff --git a/cli/src/commands/extensions/api.rs b/cli/src/commands/extensions/api.rs index f32ec0e76..953e8e8e1 100644 --- a/cli/src/commands/extensions/api.rs +++ b/cli/src/commands/extensions/api.rs @@ -1,3 +1,8 @@ +//! Most functions in this module are marked `#[allow(unused)]` for the time being. This +//! intentional, as there are no clients for those functions in the rest of the code, but +//! those functions will have to be used by the Deno integration; at that point, we may +//! remove the annotations. + use std::path::Path; use std::str::FromStr; @@ -6,6 +11,8 @@ use crate::config::{get_current_project, Config}; use crate::{api::PhylumApi, auth::UserInfo}; use anyhow::{anyhow, Context, Error, Result}; +use deno_core::OpState; +use once_cell::sync::Lazy; use phylum_types::types::auth::{AccessToken, RefreshToken}; use phylum_types::types::common::{JobId, ProjectId}; use phylum_types::types::job::JobStatusResponse; @@ -14,13 +21,35 @@ use phylum_types::types::package::{ }; use phylum_types::types::project::ProjectDetailsResponse; +/// Container for lazily evaluated dependencies of Extensions API functions. These values won't be +/// visible from extension code and will have to be set up by the JS runtime builder which will +/// load their configuration as any other command and then provide the pertinent factories. +struct InjectedDependencies { + api: Lazy, + config: Lazy, +} + +impl InjectedDependencies { + #[allow(unused)] + pub(super) fn from_factories( + api_factory: fn() -> PhylumApi, + config_factory: fn() -> Config, + ) -> Self { + InjectedDependencies { + api: Lazy::new(api_factory), + config: Lazy::new(config_factory), + } + } +} + #[allow(unused)] pub(super) async fn phylum_analyze( - api: &mut PhylumApi, + state: &mut OpState, lockfile: &str, project: Option<&str>, group: Option<&str>, ) -> Result { + let api = &mut state.borrow_mut::().api; let (packages, request_type) = get_packages_from_lockfile(Path::new(lockfile)) .context("Unable to locate any valid package in package lockfile")?; @@ -50,16 +79,21 @@ pub(super) async fn phylum_analyze( } #[allow(unused)] -pub(super) async fn phylum_auth_status(api: &mut PhylumApi, config: &Config) -> Result { - api.user_info(&config.auth_info).await.map_err(Error::from) +pub(super) async fn phylum_auth_status(state: &mut OpState) -> Result { + let deps = state.borrow_mut::(); + deps.api + .user_info(&deps.config.auth_info) + .await + .map_err(Error::from) } #[allow(unused)] pub(super) async fn phylum_auth_token_bearer( - config: &Config, + state: &mut OpState, ignore_certs: bool, ) -> Result { - let refresh_token = phylum_auth_token(config)?; + let refresh_token = phylum_auth_token(state)?; + let config = &mut state.borrow_mut::().config; let access_token = crate::auth::handle_refresh_tokens(&config.auth_info, &refresh_token, ignore_certs) .await? @@ -68,7 +102,8 @@ pub(super) async fn phylum_auth_token_bearer( } #[allow(unused)] -pub(super) fn phylum_auth_token(config: &Config) -> Result { +pub(super) fn phylum_auth_token(state: &mut OpState) -> Result { + let config = &mut state.borrow_mut::().config; config .auth_info .offline_access @@ -78,9 +113,10 @@ pub(super) fn phylum_auth_token(config: &Config) -> Result { #[allow(unused)] pub(super) async fn phylum_history_job( - api: &mut PhylumApi, + state: &mut OpState, job_id: Option<&str>, ) -> Result> { + let api = &mut state.borrow_mut::().api; let job_id = job_id .map(|job_id| JobId::from_str(job_id).ok()) .unwrap_or_else(|| get_current_project().map(|p| p.id)) @@ -90,9 +126,10 @@ pub(super) async fn phylum_history_job( #[allow(unused)] pub(super) async fn phylum_history_project( - api: &mut PhylumApi, + state: &mut OpState, project_name: Option<&str>, ) -> Result { + let api = &mut state.borrow_mut::().api; let project_name = project_name .map(String::from) .map(Result::Ok) @@ -108,11 +145,12 @@ pub(super) async fn phylum_history_project( #[allow(unused)] pub(super) async fn phylum_package( - api: &mut PhylumApi, + state: &mut OpState, name: &str, version: &str, package_type: &str, ) -> Result { + let api = &mut state.borrow_mut::().api; let package_type = PackageType::from_str(package_type) .map_err(|e| anyhow!("Unrecognized package type `{package_type}`: {e:?}"))?; api.get_package_details(&PackageDescriptor { From 5af618c99a1e0a09c968695d14e984b61bea55a8 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Tue, 24 May 2022 11:42:28 +0200 Subject: [PATCH 25/52] Use `DirBuilder` --- .cargo/config | 2 +- cli/src/commands/extensions/extension.rs | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/.cargo/config b/.cargo/config index 81baff1a3..8a1053086 100644 --- a/.cargo/config +++ b/.cargo/config @@ -1,3 +1,3 @@ [alias] xtask = "run --package xtask --" -lints = "clippy --locked --all-features --all-targets -- -D warnings" +lint = "clippy --locked --all-features --all-targets -- -D warnings" diff --git a/cli/src/commands/extensions/extension.rs b/cli/src/commands/extensions/extension.rs index 7ec69fd7f..cf33b37e7 100644 --- a/cli/src/commands/extensions/extension.rs +++ b/cli/src/commands/extensions/extension.rs @@ -1,7 +1,7 @@ use std::convert::TryFrom; -use std::fs::{self, Permissions}; +use std::fs::{self, DirBuilder}; #[cfg(unix)] -use std::os::unix::prelude::PermissionsExt; +use std::os::unix::fs::DirBuilderExt; use std::path::PathBuf; use anyhow::{anyhow, Result}; @@ -61,9 +61,14 @@ impl Extension { let dest_path = target_prefix.join(source_path.strip_prefix(&self.path)?); if source_path.is_dir() { - fs::create_dir_all(&dest_path)?; - #[cfg(unix)] - fs::set_permissions(&dest_path, Permissions::from_mode(0o700))?; + if cfg!(unix) { + DirBuilder::new() + .recursive(true) + .mode(0o700) + .create(&dest_path)?; + } else { + fs::create_dir_all(&dest_path)?; + } } else if source_path.is_file() { if dest_path.exists() { return Err(anyhow!("{}: already exists", dest_path.to_string_lossy())); From f23530c608e1d6875a39ffb8f74a517ae2b1e754 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Tue, 24 May 2022 11:44:16 +0200 Subject: [PATCH 26/52] Rename `extensions_subcommands` --- cli/src/app.rs | 2 +- cli/src/commands/extensions/mod.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/src/app.rs b/cli/src/app.rs index 5b7ffa6e1..6d55c4c27 100644 --- a/cli/src/app.rs +++ b/cli/src/app.rs @@ -185,7 +185,7 @@ pub fn app<'a>() -> clap::Command<'a> { #[cfg(feature = "extensions")] { app = app.subcommand(extensions::command()); - app = extensions::extensions_subcommands(app); + app = extensions::add_extensions_subcommands(app); } #[cfg(feature = "selfmanage")] diff --git a/cli/src/commands/extensions/mod.rs b/cli/src/commands/extensions/mod.rs index c01d64785..dc93a7c55 100644 --- a/cli/src/commands/extensions/mod.rs +++ b/cli/src/commands/extensions/mod.rs @@ -27,7 +27,7 @@ pub fn command<'a>() -> Command<'a> { /// Generate the subcommands for each extension. /// TODO add tests. -pub fn extensions_subcommands(command: Command<'_>) -> Command<'_> { +pub fn add_extensions_subcommands(command: Command<'_>) -> Command<'_> { let extensions = match installed_extensions() { Ok(extensions) => extensions, Err(e) => { From 815430a15f990e38473a7a3882c78f63aaa98b69 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Tue, 24 May 2022 11:53:42 +0200 Subject: [PATCH 27/52] Env mutex for tests --- cli/tests/extensions.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cli/tests/extensions.rs b/cli/tests/extensions.rs index 5a5697eab..5ce473852 100644 --- a/cli/tests/extensions.rs +++ b/cli/tests/extensions.rs @@ -5,12 +5,22 @@ use std::env; use std::ffi::OsStr; use std::fs; use std::path::{Path, PathBuf}; +use std::sync::Mutex; use assert_cmd::Command; +use lazy_static::lazy_static; use phylum_cli::commands::extensions::*; use rand::Rng; use regex::Regex; +lazy_static! { + // Lock this mutex when setting an environment variable, for the lifetime of function calls + // depending on that specific environment variable value. Currently only used by the + // `extension_is_installed_correctly` test. This trades some contention for the possibility + // of running tests in parallel. + static ref ENV_MUTEX: Mutex<()> = Mutex::new(()); +} + //////////////////////////////////////////////////////////////////////////////// // Acceptance criteria tests //////////////////////////////////////////////////////////////////////////////// @@ -29,6 +39,7 @@ fn extension_is_installed_correctly() { .assert() .success(); + let _guard = ENV_MUTEX.lock().unwrap(); env::set_var("XDG_DATA_HOME", &tmp_dir); let installed_ext = Extension::load("sample-extension").unwrap(); From d4c966daec221424db9a8e01d428f82c83b0e629 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Tue, 24 May 2022 12:17:22 +0200 Subject: [PATCH 28/52] Renamed, re-scoped and documented extensions API functions --- cli/src/commands/extensions/api.rs | 63 +++++++++++++++++------------- 1 file changed, 35 insertions(+), 28 deletions(-) diff --git a/cli/src/commands/extensions/api.rs b/cli/src/commands/extensions/api.rs index 953e8e8e1..bbbe1ba07 100644 --- a/cli/src/commands/extensions/api.rs +++ b/cli/src/commands/extensions/api.rs @@ -1,7 +1,7 @@ -//! Most functions in this module are marked `#[allow(unused)]` for the time being. This -//! intentional, as there are no clients for those functions in the rest of the code, but -//! those functions will have to be used by the Deno integration; at that point, we may -//! remove the annotations. +//! TODO remove the following annotation before merging. The functions in +//! this module should have as unique client the Deno runtime; pending its +//! implementation, having these functions unused would break CI. +#![allow(unused)] use std::path::Path; use std::str::FromStr; @@ -30,8 +30,7 @@ struct InjectedDependencies { } impl InjectedDependencies { - #[allow(unused)] - pub(super) fn from_factories( + pub(crate) fn from_factories( api_factory: fn() -> PhylumApi, config_factory: fn() -> Config, ) -> Self { @@ -42,8 +41,9 @@ impl InjectedDependencies { } } -#[allow(unused)] -pub(super) async fn phylum_analyze( +/// Analyze a lockfile. +/// Equivalent to `phylum analyze`. +pub(crate) async fn analyze( state: &mut OpState, lockfile: &str, project: Option<&str>, @@ -78,31 +78,31 @@ pub(super) async fn phylum_analyze( Ok(job_id) } -#[allow(unused)] -pub(super) async fn phylum_auth_status(state: &mut OpState) -> Result { +/// Retrieve user info. +/// Equivalent to `phylum auth status`. +pub(crate) async fn get_user_info(state: &mut OpState) -> Result { let deps = state.borrow_mut::(); - deps.api - .user_info(&deps.config.auth_info) - .await - .map_err(Error::from) + deps.api.user_info().await.map_err(Error::from) } -#[allow(unused)] -pub(super) async fn phylum_auth_token_bearer( +/// Retrieve the access token. +/// Equivalent to `phylum auth token --bearer`. +pub(crate) async fn get_access_token( state: &mut OpState, ignore_certs: bool, ) -> Result { - let refresh_token = phylum_auth_token(state)?; + let refresh_token = get_refresh_token(state)?; let config = &mut state.borrow_mut::().config; let access_token = - crate::auth::handle_refresh_tokens(&config.auth_info, &refresh_token, ignore_certs) + crate::auth::handle_refresh_tokens(&refresh_token, ignore_certs, &config.connection.uri) .await? .access_token; Ok(access_token) } -#[allow(unused)] -pub(super) fn phylum_auth_token(state: &mut OpState) -> Result { +/// Retrieve the refresh token. +/// Equivalent to `phylum auth token`. +pub(crate) fn get_refresh_token(state: &mut OpState) -> Result { let config = &mut state.borrow_mut::().config; config .auth_info @@ -111,8 +111,9 @@ pub(super) fn phylum_auth_token(state: &mut OpState) -> Result { .ok_or_else(|| anyhow!("User is not currently authenticated")) } -#[allow(unused)] -pub(super) async fn phylum_history_job( +/// Retrieve a job's status. +/// Equivalent to `phylum history job`. +pub(crate) async fn get_job_status( state: &mut OpState, job_id: Option<&str>, ) -> Result> { @@ -124,8 +125,9 @@ pub(super) async fn phylum_history_job( api.get_job_status_ext(&job_id).await.map_err(Error::from) } -#[allow(unused)] -pub(super) async fn phylum_history_project( +/// Retrieve a project's details. +/// Equivalent to `phylum history project`. +pub(crate) async fn get_project_details( state: &mut OpState, project_name: Option<&str>, ) -> Result { @@ -143,8 +145,9 @@ pub(super) async fn phylum_history_project( .map_err(Error::from) } -#[allow(unused)] -pub(super) async fn phylum_package( +/// Analyze a single package. +/// Equivalent to `phylum package`. +pub(crate) async fn analyze_package( state: &mut OpState, name: &str, version: &str, @@ -162,8 +165,12 @@ pub(super) async fn phylum_package( .map_err(Error::from) } -#[allow(unused)] -pub(super) fn phylum_parse(lockfile: &str, lockfile_type: &str) -> Result> { +/// Parse a lockfile and return the package descriptors contained therein. +/// Equivalent to `phylum parse`. +pub(crate) fn parse_lockfile( + lockfile: &str, + lockfile_type: &str, +) -> Result> { let parser = LOCKFILE_PARSERS .iter() .find_map(|(name, parser)| (*name == lockfile_type).then(|| parser)) From 1d1833f324f70902a953ca8fdd2a2af1a6fbab4a Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Tue, 24 May 2022 17:13:09 +0200 Subject: [PATCH 29/52] Handle run extension placeholder, refactor if/else chain in binary to non-exhaustive match --- cli/src/bin/phylum.rs | 45 ++++++++++++++++++++---------- cli/src/commands/extensions/mod.rs | 11 ++++++++ 2 files changed, 42 insertions(+), 14 deletions(-) diff --git a/cli/src/bin/phylum.rs b/cli/src/bin/phylum.rs index 11dbd4bdc..1ef40503b 100644 --- a/cli/src/bin/phylum.rs +++ b/cli/src/bin/phylum.rs @@ -196,23 +196,40 @@ async fn handle_commands() -> CommandResult { return Ok(ExitCode::Ok.into()); } - let should_submit = matches.subcommand_matches("analyze").is_some() - || matches.subcommand_matches("batch").is_some(); - // TODO: switch from if/else to non-exhaustive pattern match - if let Some(matches) = matches.subcommand_matches("project") { - handle_project(&mut api, matches).await?; - } else if let Some(matches) = matches.subcommand_matches("package") { - return handle_get_package(&mut api, &config.request_type, matches).await; - } else if should_submit { - return handle_submission(&mut api, config, &matches).await; - } else if let Some(matches) = matches.subcommand_matches("history") { - return handle_history(&mut api, matches).await; - } else if let Some(matches) = matches.subcommand_matches("group") { - return handle_group(&mut api, matches).await; + match matches.subcommand() { + Some(("project", matches)) => handle_project(&mut api, matches).await, + Some(("package", matches)) => { + handle_get_package(&mut api, &config.request_type, matches).await + } + Some(("history", matches)) => handle_submission(&mut api, config, matches).await, + Some(("group", matches)) => handle_group(&mut api, matches).await, + Some(("analyze", _)) | Some(("batch", _)) => { + handle_submission(&mut api, config, &matches).await + } + Some((extension, matches)) => handle_run_extension(extension, matches).await, + None => unreachable!() } - Ok(ExitCode::Ok.into()) + // Drop this code after we have validated that the above match statement fulfills + // everything from the following lines. + + // let should_submit = matches.subcommand_matches("analyze").is_some() + // || matches.subcommand_matches("batch").is_some(); + // + // if let Some(matches) = matches.subcommand_matches("project") { + // handle_project(&mut api, matches).await?; + // } else if let Some(matches) = matches.subcommand_matches("package") { + // return handle_get_package(&mut api, &config.request_type, matches).await; + // } else if should_submit { + // return handle_submission(&mut api, config, &matches).await; + // } else if let Some(matches) = matches.subcommand_matches("history") { + // return handle_history(&mut api, matches).await; + // } else if let Some(matches) = matches.subcommand_matches("group") { + // return handle_group(&mut api, matches).await; + // } + + // Ok(ExitCode::Ok.into()) } #[tokio::main] diff --git a/cli/src/commands/extensions/mod.rs b/cli/src/commands/extensions/mod.rs index ba910dc06..cfd52ca2b 100644 --- a/cli/src/commands/extensions/mod.rs +++ b/cli/src/commands/extensions/mod.rs @@ -108,6 +108,17 @@ async fn handle_list_extensions() -> CommandResult { Ok(CommandValue::Code(ExitCode::Ok)) } +/// Handle running an extension. +pub async fn handle_run_extension(extension: &str, matches: &ArgMatches) -> CommandResult { + println!("{}", extension); + println!("{:?}", matches); + + let ext = Extension::load(extension)?; + println!("{:?}", ext); + + Ok(CommandValue::Code(ExitCode::Ok)) +} + // Return a list of installed extensions. Filter out invalid extensions instead of exiting early. fn installed_extensions() -> Result> { let extensions_path = extensions_path()?; From 5f7e0ba55e29c4d275a8c14ca64529ccc00d4463 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Tue, 24 May 2022 17:24:09 +0200 Subject: [PATCH 30/52] Changed extension name regex --- cli/src/commands/extensions/extension.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/src/commands/extensions/extension.rs b/cli/src/commands/extensions/extension.rs index cf33b37e7..dc13b303b 100644 --- a/cli/src/commands/extensions/extension.rs +++ b/cli/src/commands/extensions/extension.rs @@ -13,7 +13,7 @@ use walkdir::WalkDir; const MANIFEST_NAME: &str = "PhylumExt.toml"; lazy_static! { - static ref EXTENSION_NAME_RE: Regex = Regex::new(r#"^[a-z0-9_-]+$"#).unwrap(); + static ref EXTENSION_NAME_RE: Regex = Regex::new(r#"^[a-z][a-z0-9-]+$"#).unwrap(); } #[derive(Debug)] From 35a9708be7cddcf2092cbd0a14796f4a3e994fa4 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Tue, 31 May 2022 16:07:48 +0200 Subject: [PATCH 31/52] Adapted Parse::parse_file to allow trait objects --- cli/src/commands/extensions/api.rs | 26 +++++++++++++------------- cli/src/lockfiles/csharp.rs | 8 ++++++-- cli/src/lockfiles/java.rs | 8 ++++++-- cli/src/lockfiles/javascript.rs | 16 ++++++++++------ cli/src/lockfiles/mod.rs | 5 +---- cli/src/lockfiles/python.rs | 22 ++++++++++++++++------ cli/src/lockfiles/ruby.rs | 6 +++++- 7 files changed, 57 insertions(+), 34 deletions(-) diff --git a/cli/src/commands/extensions/api.rs b/cli/src/commands/extensions/api.rs index c82d7554b..2f1dde5fd 100644 --- a/cli/src/commands/extensions/api.rs +++ b/cli/src/commands/extensions/api.rs @@ -166,16 +166,16 @@ pub(crate) async fn analyze_package( .map_err(Error::from) } -// Parse a lockfile and return the package descriptors contained therein. -// Equivalent to `phylum parse`. -// pub(crate) fn parse_lockfile( -// lockfile: &str, -// lockfile_type: &str, -// ) -> Result> { -// let parser = LOCKFILE_PARSERS -// .iter() -// .find_map(|(name, parser)| (*name == lockfile_type).then(|| *parser)) -// .ok_or_else(|| anyhow!("Unrecognized lockfile type: `{lockfile_type}`"))?; -// -// parser.parse_file(Path::new(lockfile)) -// } +/// Parse a lockfile and return the package descriptors contained therein. +/// Equivalent to `phylum parse`. +pub(crate) fn parse_lockfile( + lockfile: &str, + lockfile_type: &str, +) -> Result> { + let parser = LOCKFILE_PARSERS + .iter() + .find_map(|(name, parser)| (*name == lockfile_type).then(|| *parser)) + .ok_or_else(|| anyhow!("Unrecognized lockfile type: `{lockfile_type}`"))?; + + parser.parse_file(Path::new(lockfile)) +} diff --git a/cli/src/lockfiles/csharp.rs b/cli/src/lockfiles/csharp.rs index 9610c4558..5b59db179 100644 --- a/cli/src/lockfiles/csharp.rs +++ b/cli/src/lockfiles/csharp.rs @@ -76,11 +76,15 @@ impl Parse for CSProj { #[cfg(test)] mod tests { + use std::path::Path; + use super::*; #[test] fn lock_parse_csproj() { - let pkgs = CSProj.parse_file("tests/fixtures/sample.csproj").unwrap(); + let pkgs = CSProj + .parse_file(Path::new("tests/fixtures/sample.csproj")) + .unwrap(); assert_eq!(pkgs.len(), 5); assert_eq!(pkgs[0].name, "Microsoft.NETFramework.ReferenceAssemblies"); @@ -96,7 +100,7 @@ mod tests { #[test] fn lock_parse_another_invalid_char() { let pkgs = CSProj - .parse_file("tests/fixtures/Calculator.csproj") + .parse_file(Path::new("tests/fixtures/Calculator.csproj")) .unwrap(); assert!(!pkgs.is_empty()); } diff --git a/cli/src/lockfiles/java.rs b/cli/src/lockfiles/java.rs index a9af772ab..b842fd438 100644 --- a/cli/src/lockfiles/java.rs +++ b/cli/src/lockfiles/java.rs @@ -136,12 +136,14 @@ impl Parse for Pom { #[cfg(test)] mod tests { + use std::path::Path; + use super::*; #[test] fn lock_parse_gradle() { let pkgs = GradleLock - .parse_file("tests/fixtures/gradle.lockfile") + .parse_file(Path::new("tests/fixtures/gradle.lockfile")) .unwrap(); assert_eq!(pkgs.len(), 5); @@ -157,7 +159,9 @@ mod tests { #[test] fn lock_parse_effective_pom() { - let mut pkgs = Pom.parse_file("tests/fixtures/effective-pom.xml").unwrap(); + let mut pkgs = Pom + .parse_file(Path::new("tests/fixtures/effective-pom.xml")) + .unwrap(); pkgs.sort_by(|a, b| a.version.cmp(&b.version)); assert_eq!(pkgs.len(), 16); diff --git a/cli/src/lockfiles/javascript.rs b/cli/src/lockfiles/javascript.rs index c89aa75bc..970aeaf46 100644 --- a/cli/src/lockfiles/javascript.rs +++ b/cli/src/lockfiles/javascript.rs @@ -167,12 +167,14 @@ impl Parse for YarnLock { #[cfg(test)] mod tests { + use std::path::Path; + use super::*; #[test] fn lock_parse_package() { let pkgs = PackageLock - .parse_file("tests/fixtures/package-lock-v6.json") + .parse_file(Path::new("tests/fixtures/package-lock-v6.json")) .unwrap(); assert_eq!(pkgs.len(), 17); @@ -189,7 +191,7 @@ mod tests { #[test] fn lock_parse_package_v7() { let pkgs = PackageLock - .parse_file("tests/fixtures/package-lock.json") + .parse_file(Path::new("tests/fixtures/package-lock.json")) .unwrap(); assert_eq!(pkgs.len(), 50); @@ -219,7 +221,7 @@ mod tests { // We need to make sure we don't take the v2 lockfile code path because this is not a v2 // lockfile and parsing it as one will produce incorrect results. let pkgs = YarnLock - .parse_file("tests/fixtures/yarn-v1.simple.lock") + .parse_file(Path::new("tests/fixtures/yarn-v1.simple.lock")) .unwrap(); assert_eq!( @@ -238,7 +240,7 @@ mod tests { "tests/fixtures/yarn-v1.lock", "tests/fixtures/yarn-v1.trailing_newlines.lock", ] { - let pkgs = YarnLock.parse_file(p).unwrap(); + let pkgs = YarnLock.parse_file(Path::new(p)).unwrap(); assert_eq!(pkgs.len(), 17); @@ -261,13 +263,15 @@ mod tests { #[test] fn lock_parse_yarn_v1_malformed_fails() { YarnLock - .parse_file("tests/fixtures/yarn-v1.lock.bad") + .parse_file(Path::new("tests/fixtures/yarn-v1.lock.bad")) .unwrap(); } #[test] fn lock_parse_yarn() { - let pkgs = YarnLock.parse_file("tests/fixtures/yarn.lock").unwrap(); + let pkgs = YarnLock + .parse_file(Path::new("tests/fixtures/yarn.lock")) + .unwrap(); assert_eq!(pkgs.len(), 53); diff --git a/cli/src/lockfiles/mod.rs b/cli/src/lockfiles/mod.rs index 56a55146e..944fb5672 100644 --- a/cli/src/lockfiles/mod.rs +++ b/cli/src/lockfiles/mod.rs @@ -24,10 +24,7 @@ pub trait Parse { fn parse(&self, data: &str) -> ParseResult; /// Parse from a file - fn parse_file>(&self, path: P) -> ParseResult - where - Self: Sized, - { + fn parse_file(&self, path: &Path) -> ParseResult { let data = read_to_string(path)?; self.parse(&data) } diff --git a/cli/src/lockfiles/python.rs b/cli/src/lockfiles/python.rs index d5b1b5af3..ac40ea113 100644 --- a/cli/src/lockfiles/python.rs +++ b/cli/src/lockfiles/python.rs @@ -167,12 +167,14 @@ struct PoetryMetadata { #[cfg(test)] mod tests { + use std::path::Path; + use super::*; #[test] fn parse_requirements() { let pkgs = PyRequirements - .parse_file("tests/fixtures/requirements.txt") + .parse_file(Path::new("tests/fixtures/requirements.txt")) .unwrap(); assert_eq!(pkgs.len(), 131); assert_eq!(pkgs[0].name, "pyyaml"); @@ -188,7 +190,7 @@ mod tests { #[test] fn parse_requirements_complex() { let pkgs = PyRequirements - .parse_file("tests/fixtures/complex-requirements.txt") + .parse_file(Path::new("tests/fixtures/complex-requirements.txt")) .unwrap(); assert_eq!(pkgs.len(), 8); assert_eq!(pkgs[0].name, "docopt"); @@ -206,7 +208,9 @@ mod tests { #[test] fn parse_pipfile() { - let pkgs = PipFile.parse_file("tests/fixtures/Pipfile").unwrap(); + let pkgs = PipFile + .parse_file(Path::new("tests/fixtures/Pipfile")) + .unwrap(); assert_eq!(pkgs.len(), 4); let expected_pkgs = [ @@ -234,7 +238,9 @@ mod tests { #[test] fn lock_parse_pipfile() { - let pkgs = PipFile.parse_file("tests/fixtures/Pipfile.lock").unwrap(); + let pkgs = PipFile + .parse_file(Path::new("tests/fixtures/Pipfile.lock")) + .unwrap(); assert_eq!(pkgs.len(), 27); let expected_pkgs = [ @@ -262,7 +268,9 @@ mod tests { #[test] fn parse_poetry_lock() { - let pkgs = Poetry.parse_file("tests/fixtures/poetry.lock").unwrap(); + let pkgs = Poetry + .parse_file(Path::new("tests/fixtures/poetry.lock")) + .unwrap(); assert_eq!(pkgs.len(), 44); let expected_pkgs = [ @@ -291,7 +299,9 @@ mod tests { /// Ensure sources other than PyPi are ignored. #[test] fn poetry_ignore_other_sources() { - let pkgs = Poetry.parse_file("tests/fixtures/poetry.lock").unwrap(); + let pkgs = Poetry + .parse_file(Path::new("tests/fixtures/poetry.lock")) + .unwrap(); let invalid_package_names = ["toml", "directory-test", "requests"]; for pkg in pkgs { diff --git a/cli/src/lockfiles/ruby.rs b/cli/src/lockfiles/ruby.rs index 04620a67d..e3d1df641 100644 --- a/cli/src/lockfiles/ruby.rs +++ b/cli/src/lockfiles/ruby.rs @@ -25,13 +25,17 @@ impl Parse for GemLock { #[cfg(test)] mod tests { + use std::path::Path; + use phylum_types::types::package::PackageType; use super::*; #[test] fn lock_parse_gem() { - let pkgs = GemLock.parse_file("tests/fixtures/Gemfile.lock").unwrap(); + let pkgs = GemLock + .parse_file(Path::new("tests/fixtures/Gemfile.lock")) + .unwrap(); assert_eq!(pkgs.len(), 214); assert_eq!(pkgs[0].name, "CFPropertyList"); assert_eq!(pkgs[0].version, "2.3.6"); From 5701365c2653f68344a74d0b6c685cd8d6c4097e Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Tue, 31 May 2022 17:17:02 +0200 Subject: [PATCH 32/52] Removed unused `handle_run_extension` function --- cli/src/bin/phylum.rs | 1 + cli/src/commands/extensions/mod.rs | 11 ----------- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/cli/src/bin/phylum.rs b/cli/src/bin/phylum.rs index c840dbf31..b6b9d49c5 100644 --- a/cli/src/bin/phylum.rs +++ b/cli/src/bin/phylum.rs @@ -214,6 +214,7 @@ async fn handle_ping(mut api: PhylumApi) -> CommandResult { print_response(&resp, true, None); Ok(ExitCode::Ok.into()) } + async fn handle_update(matches: &ArgMatches) -> CommandResult { let mut spinner = Spinner::new( Spinners::Dots12, diff --git a/cli/src/commands/extensions/mod.rs b/cli/src/commands/extensions/mod.rs index a3f3739a3..ea02b4344 100644 --- a/cli/src/commands/extensions/mod.rs +++ b/cli/src/commands/extensions/mod.rs @@ -108,17 +108,6 @@ async fn handle_list_extensions() -> CommandResult { Ok(CommandValue::Code(ExitCode::Ok)) } -/// Handle running an extension. -pub async fn handle_run_extension(extension: &str, matches: &ArgMatches) -> CommandResult { - println!("{}", extension); - println!("{:?}", matches); - - let ext = Extension::load(extension)?; - println!("{:?}", ext); - - Ok(CommandValue::Code(ExitCode::Ok)) -} - /// Return a list of installed extensions. Filter out invalid extensions instead of exiting early. pub fn installed_extensions() -> Result> { let extensions_path = extensions_path()?; From 5192642a8ce36f5967fbc0110420f2807e47c542 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Wed, 1 Jun 2022 15:18:24 +0200 Subject: [PATCH 33/52] Made API methods require const ref, created deno ops --- cli/src/api/mod.rs | 49 +++++++++------------- cli/src/bin/phylum.rs | 2 +- cli/src/commands/extensions/api.rs | 66 ++++++++++++++++++++---------- 3 files changed, 66 insertions(+), 51 deletions(-) diff --git a/cli/src/api/mod.rs b/cli/src/api/mod.rs index 0c3bf62d2..e1570e5c3 100644 --- a/cli/src/api/mod.rs +++ b/cli/src/api/mod.rs @@ -171,7 +171,7 @@ impl PhylumApi { } /// Ping the system and verify it's up - pub async fn ping(&mut self) -> Result { + pub async fn ping(&self) -> Result { Ok(self .get::(endpoints::get_ping(&self.api_uri)) .await? @@ -185,7 +185,7 @@ impl PhylumApi { } /// Create a new project - pub async fn create_project(&mut self, name: &str, group: Option<&str>) -> Result { + pub async fn create_project(&self, name: &str, group: Option<&str>) -> Result { Ok(self .put::( endpoints::put_create_project(&self.api_uri), @@ -199,10 +199,7 @@ impl PhylumApi { } /// Get a list of projects - pub async fn get_projects( - &mut self, - group: Option<&str>, - ) -> Result> { + pub async fn get_projects(&self, group: Option<&str>) -> Result> { let uri = match group { Some(group) => endpoints::group_project_summary(&self.api_uri, group), None => endpoints::get_project_summary(&self.api_uri), @@ -212,12 +209,12 @@ impl PhylumApi { } /// Get user settings - pub async fn get_user_settings(&mut self) -> Result { + pub async fn get_user_settings(&self) -> Result { self.get(endpoints::get_user_settings(&self.api_uri)).await } /// Put updated user settings - pub async fn put_user_settings(&mut self, settings: &UserSettings) -> Result { + pub async fn put_user_settings(&self, settings: &UserSettings) -> Result { self.put::(endpoints::put_user_settings(&self.api_uri), &settings) .await?; Ok(true) @@ -225,7 +222,7 @@ impl PhylumApi { /// Submit a new request to the system pub async fn submit_request( - &mut self, + &self, req_type: &PackageType, package_list: &[PackageDescriptor], is_user: bool, @@ -249,17 +246,14 @@ impl PhylumApi { } /// Get the status of a previously submitted job - pub async fn get_job_status( - &mut self, - job_id: &JobId, - ) -> Result> { + pub async fn get_job_status(&self, job_id: &JobId) -> Result> { self.get(endpoints::get_job_status(&self.api_uri, job_id, false)) .await } /// Get the status of a previously submitted job (verbose output) pub async fn get_job_status_ext( - &mut self, + &self, job_id: &JobId, ) -> Result> { self.get(endpoints::get_job_status(&self.api_uri, job_id, true)) @@ -267,23 +261,20 @@ impl PhylumApi { } /// Get the status of all jobs - pub async fn get_status(&mut self) -> Result { + pub async fn get_status(&self) -> Result { self.get(endpoints::get_all_jobs_status(&self.api_uri, 30)) .await } /// Get the details of a specific project - pub async fn get_project_details( - &mut self, - project_name: &str, - ) -> Result { + pub async fn get_project_details(&self, project_name: &str) -> Result { self.get(endpoints::get_project_details(&self.api_uri, project_name)) .await } /// Resolve a Project Name to a Project ID pub async fn get_project_id( - &mut self, + &self, project_name: &str, group_name: Option<&str>, ) -> Result { @@ -303,18 +294,18 @@ impl PhylumApi { } /// Get package details - pub async fn get_package_details(&mut self, pkg: &PackageDescriptor) -> Result { + pub async fn get_package_details(&self, pkg: &PackageDescriptor) -> Result { self.get(endpoints::get_package_status(&self.api_uri, pkg)) .await } /// Get all groups the user is part of. - pub async fn get_groups_list(&mut self) -> Result { + pub async fn get_groups_list(&self) -> Result { self.get(endpoints::group_list(&self.api_uri)).await } /// Get all groups the user is part of. - pub async fn create_group(&mut self, group_name: &str) -> Result { + pub async fn create_group(&self, group_name: &str) -> Result { let group = CreateGroupRequest { group_name: group_name.into(), }; @@ -382,7 +373,7 @@ mod tests { .mount(&mock_server) .await; - let mut client = build_phylum_api(&mock_server).await?; + let client = build_phylum_api(&mock_server).await?; let pkg = PackageDescriptor { name: "react".to_string(), @@ -414,7 +405,7 @@ mod tests { .mount(&mock_server) .await; - let mut client = build_phylum_api(&mock_server).await?; + let client = build_phylum_api(&mock_server).await?; let pkg = PackageDescriptor { name: "react".to_string(), @@ -489,7 +480,7 @@ mod tests { .mount(&mock_server) .await; - let mut client = build_phylum_api(&mock_server).await?; + let client = build_phylum_api(&mock_server).await?; client.get_status().await?; Ok(()) } @@ -547,7 +538,7 @@ mod tests { .mount(&mock_server) .await; - let mut client = build_phylum_api(&mock_server).await?; + let client = build_phylum_api(&mock_server).await?; let pkg = PackageDescriptor { name: "@schematics/angular".to_string(), @@ -605,7 +596,7 @@ mod tests { .mount(&mock_server) .await; - let mut client = build_phylum_api(&mock_server).await?; + let client = build_phylum_api(&mock_server).await?; let job = JobId::from_str("59482a54-423b-448d-8325-f171c9dc336b").unwrap(); client.get_job_status(&job).await?; @@ -681,7 +672,7 @@ mod tests { .mount(&mock_server) .await; - let mut client = build_phylum_api(&mock_server).await?; + let client = build_phylum_api(&mock_server).await?; let job = JobId::from_str("59482a54-423b-448d-8325-f171c9dc336b").unwrap(); client.get_job_status_ext(&job).await?; diff --git a/cli/src/bin/phylum.rs b/cli/src/bin/phylum.rs index 062711033..1065f8eb3 100644 --- a/cli/src/bin/phylum.rs +++ b/cli/src/bin/phylum.rs @@ -201,7 +201,7 @@ async fn handle_commands() -> CommandResult { } } -async fn handle_ping(mut api: PhylumApi) -> CommandResult { +async fn handle_ping(api: PhylumApi) -> CommandResult { let resp = api.ping().await; print_response(&resp, true, None); Ok(ExitCode::Ok.into()) diff --git a/cli/src/commands/extensions/api.rs b/cli/src/commands/extensions/api.rs index 2f1dde5fd..4a809dbe5 100644 --- a/cli/src/commands/extensions/api.rs +++ b/cli/src/commands/extensions/api.rs @@ -3,8 +3,11 @@ //! implementation, having these functions unused would break CI. #![allow(unused)] +use std::cell::RefCell; use std::path::Path; +use std::rc::Rc; use std::str::FromStr; +use std::sync::Arc; use crate::commands::parse::{get_packages_from_lockfile, LOCKFILE_PARSERS}; use crate::config::{get_current_project, Config}; @@ -12,7 +15,8 @@ use crate::lockfiles::Parse; use crate::{api::PhylumApi, auth::UserInfo}; use anyhow::{anyhow, Context, Error, Result}; -use deno_core::OpState; +use deno_core::parking_lot::{MappedMutexGuard, Mutex, MutexGuard}; +use deno_core::{op, OpState}; use once_cell::sync::Lazy; use phylum_types::types::auth::{AccessToken, RefreshToken}; use phylum_types::types::common::{JobId, ProjectId}; @@ -26,8 +30,8 @@ use phylum_types::types::project::ProjectDetailsResponse; /// visible from extension code and will have to be set up by the JS runtime builder which will /// load their configuration as any other command and then provide the pertinent factories. struct InjectedDependencies { - api: Lazy, - config: Lazy, + api: Arc>, + config: Arc>, } impl InjectedDependencies { @@ -36,21 +40,22 @@ impl InjectedDependencies { config_factory: fn() -> Config, ) -> Self { InjectedDependencies { - api: Lazy::new(api_factory), - config: Lazy::new(config_factory), + api: Arc::new(Lazy::new(api_factory)), + config: Arc::new(Lazy::new(config_factory)), } } } /// Analyze a lockfile. /// Equivalent to `phylum analyze`. +#[op] pub(crate) async fn analyze( - state: &mut OpState, + state: Rc>, lockfile: &str, project: Option<&str>, group: Option<&str>, ) -> Result { - let api = &mut state.borrow_mut::().api; + let api = { &state.borrow().borrow::().api.clone() }; let (packages, request_type) = get_packages_from_lockfile(Path::new(lockfile)) .context("Unable to locate any valid package in package lockfile")?; @@ -81,19 +86,27 @@ pub(crate) async fn analyze( /// Retrieve user info. /// Equivalent to `phylum auth status`. -pub(crate) async fn get_user_info(state: &mut OpState) -> Result { - let deps = state.borrow_mut::(); - deps.api.user_info().await.map_err(Error::from) +#[op] +pub(crate) async fn get_user_info(state: Rc>) -> Result { + let api = { &state.borrow().borrow::().api.clone() }; + api.user_info().await.map_err(Error::from) } /// Retrieve the access token. /// Equivalent to `phylum auth token --bearer`. +#[op] pub(crate) async fn get_access_token( - state: &mut OpState, + state: Rc>, ignore_certs: bool, ) -> Result { - let refresh_token = get_refresh_token(state)?; - let config = &mut state.borrow_mut::().config; + let refresh_token = get_refresh_token::call(state.clone())?; + let config = { + &state + .borrow() + .borrow::() + .config + .clone() + }; let access_token = crate::auth::handle_refresh_tokens(&refresh_token, ignore_certs, &config.connection.uri) .await? @@ -103,8 +116,15 @@ pub(crate) async fn get_access_token( /// Retrieve the refresh token. /// Equivalent to `phylum auth token`. -pub(crate) fn get_refresh_token(state: &mut OpState) -> Result { - let config = &mut state.borrow_mut::().config; +#[op] +pub(crate) fn get_refresh_token(state: Rc>) -> Result { + let config = { + &state + .borrow() + .borrow::() + .config + .clone() + }; config .auth_info .offline_access @@ -114,11 +134,12 @@ pub(crate) fn get_refresh_token(state: &mut OpState) -> Result { /// Retrieve a job's status. /// Equivalent to `phylum history job`. +#[op] pub(crate) async fn get_job_status( - state: &mut OpState, + state: Rc>, job_id: Option<&str>, ) -> Result> { - let api = &mut state.borrow_mut::().api; + let api = { &state.borrow().borrow::().api.clone() }; let job_id = job_id .map(|job_id| JobId::from_str(job_id).ok()) .unwrap_or_else(|| get_current_project().map(|p| p.id)) @@ -128,11 +149,12 @@ pub(crate) async fn get_job_status( /// Retrieve a project's details. /// Equivalent to `phylum history project`. +#[op] pub(crate) async fn get_project_details( - state: &mut OpState, + state: Rc>, project_name: Option<&str>, ) -> Result { - let api = &mut state.borrow_mut::().api; + let api = { &state.borrow().borrow::().api.clone() }; let project_name = project_name .map(String::from) .map(Result::Ok) @@ -148,13 +170,14 @@ pub(crate) async fn get_project_details( /// Analyze a single package. /// Equivalent to `phylum package`. +#[op] pub(crate) async fn analyze_package( - state: &mut OpState, + state: Rc>, name: &str, version: &str, package_type: &str, ) -> Result { - let api = &mut state.borrow_mut::().api; + let api = { &state.borrow().borrow::().api.clone() }; let package_type = PackageType::from_str(package_type) .map_err(|e| anyhow!("Unrecognized package type `{package_type}`: {e:?}"))?; api.get_package_details(&PackageDescriptor { @@ -168,6 +191,7 @@ pub(crate) async fn analyze_package( /// Parse a lockfile and return the package descriptors contained therein. /// Equivalent to `phylum parse`. +#[op] pub(crate) fn parse_lockfile( lockfile: &str, lockfile_type: &str, From 5a99cfbdcb8bf5176557c3d499c761df20b36f55 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Thu, 2 Jun 2022 17:18:50 +0200 Subject: [PATCH 34/52] BROKEN: added ops to Deno --- cli/src/api/mod.rs | 2 + cli/src/bin/phylum.rs | 12 +- cli/src/commands/extensions/api.rs | 109 +++++++++++++----- cli/src/commands/extensions/extension.rs | 16 ++- cli/src/commands/extensions/mod.rs | 18 ++- cli/src/config.rs | 6 +- cli/src/deno.rs | 20 ++-- cli/tests/extensions.rs | 23 ++++ .../extensions/api-extension/PhylumExt.toml | 3 + .../fixtures/extensions/api-extension/main.ts | 3 + 10 files changed, 167 insertions(+), 45 deletions(-) create mode 100644 cli/tests/fixtures/extensions/api-extension/PhylumExt.toml create mode 100644 cli/tests/fixtures/extensions/api-extension/main.ts diff --git a/cli/src/api/mod.rs b/cli/src/api/mod.rs index e1570e5c3..3942e61cf 100644 --- a/cli/src/api/mod.rs +++ b/cli/src/api/mod.rs @@ -25,11 +25,13 @@ use crate::auth::handle_auth_flow; use crate::auth::handle_refresh_tokens; use crate::auth::{AuthAction, UserInfo}; use crate::config::AuthInfo; +use crate::config::Config; use crate::types::PingResponse; type Result = std::result::Result; pub struct PhylumApi { + config: Config, client: Client, api_uri: String, ignore_certs: bool, diff --git a/cli/src/bin/phylum.rs b/cli/src/bin/phylum.rs index 1065f8eb3..86ac5d53a 100644 --- a/cli/src/bin/phylum.rs +++ b/cli/src/bin/phylum.rs @@ -1,4 +1,5 @@ use std::path::{Path, PathBuf}; +use std::sync::Arc; use std::time::UNIX_EPOCH; use anyhow::{anyhow, Context, Result}; @@ -11,7 +12,7 @@ use spinners::{Spinner, Spinners}; use phylum_cli::api::PhylumApi; use phylum_cli::commands::auth::*; #[cfg(feature = "extensions")] -use phylum_cli::commands::extensions::{handle_extensions, Extension}; +use phylum_cli::commands::extensions::{handle_extensions, handle_run_extension}; use phylum_cli::commands::group::handle_group; use phylum_cli::commands::jobs::*; use phylum_cli::commands::packages::*; @@ -194,7 +195,14 @@ async fn handle_commands() -> CommandResult { "extension" => handle_extensions(matches).await, #[cfg(feature = "extensions")] - extension_subcmd => Extension::load(extension_subcmd)?.run().await, + extension_subcmd => { + handle_run_extension( + extension_subcmd, + config.clone(), + Box::pin(async { api.await.map(Arc::new) }), + ) + .await + } // Extension::load(extension_subcmd)?.run().await, #[cfg(not(feature = "extensions"))] _ => unreachable!(), diff --git a/cli/src/commands/extensions/api.rs b/cli/src/commands/extensions/api.rs index 4a809dbe5..5e748f5e5 100644 --- a/cli/src/commands/extensions/api.rs +++ b/cli/src/commands/extensions/api.rs @@ -4,7 +4,9 @@ #![allow(unused)] use std::cell::RefCell; +use std::future::Future; use std::path::Path; +use std::pin::Pin; use std::rc::Rc; use std::str::FromStr; use std::sync::Arc; @@ -16,7 +18,8 @@ use crate::{api::PhylumApi, auth::UserInfo}; use anyhow::{anyhow, Context, Error, Result}; use deno_core::parking_lot::{MappedMutexGuard, Mutex, MutexGuard}; -use deno_core::{op, OpState}; +use deno_core::{op, OpDecl, OpState}; +use futures::future::BoxFuture; use once_cell::sync::Lazy; use phylum_types::types::auth::{AccessToken, RefreshToken}; use phylum_types::types::common::{JobId, ProjectId}; @@ -26,24 +29,61 @@ use phylum_types::types::package::{ }; use phylum_types::types::project::ProjectDetailsResponse; +// BROKEN + +enum OnceFuture { + Future(Option>>), + Awaited(T), +} + +impl OnceFuture { + fn new(t: BoxFuture<'static, T>) -> Self { + OnceFuture::Future(Some(t)) + } + + async fn try_get(&mut self) -> Result<&T> { + match *self { + OnceFuture::Future(Some(ref mut t)) => { + *self = OnceFuture::Awaited(t.await?); + match *self { + OnceFuture::Future(..) => unreachable!(), + OnceFuture::Awaited(ref mut t) => Ok(t), + } + } + OnceFuture::Awaited(ref mut t) => Ok(t) + } + } +} + +type PhylumApiFut = OnceFuture>; + /// Container for lazily evaluated dependencies of Extensions API functions. These values won't be /// visible from extension code and will have to be set up by the JS runtime builder which will /// load their configuration as any other command and then provide the pertinent factories. -struct InjectedDependencies { - api: Arc>, - config: Arc>, +pub(crate) struct InjectedDependencies { + api: OnceFuture>>, + config: Config, } impl InjectedDependencies { - pub(crate) fn from_factories( - api_factory: fn() -> PhylumApi, - config_factory: fn() -> Config, + pub(crate) async fn from_factories( + api_factory: BoxFuture<'static, Result>>, + config: Config, ) -> Self { InjectedDependencies { - api: Arc::new(Lazy::new(api_factory)), - config: Arc::new(Lazy::new(config_factory)), + api: OnceFuture::new(api_factory), + config, } } + + async fn api(&mut self) -> Result> { + self.api + .get() + .await + .as_ref() + .cloned() + .map_err(|e| anyhow!("{:?}", e)) + } } /// Analyze a lockfile. @@ -55,7 +95,9 @@ pub(crate) async fn analyze( project: Option<&str>, group: Option<&str>, ) -> Result { - let api = { &state.borrow().borrow::().api.clone() }; + let mut state = Pin::new(state.borrow_mut()); + let api = state.borrow_mut::().get().await?; + let (packages, request_type) = get_packages_from_lockfile(Path::new(lockfile)) .context("Unable to locate any valid package in package lockfile")?; @@ -88,7 +130,8 @@ pub(crate) async fn analyze( /// Equivalent to `phylum auth status`. #[op] pub(crate) async fn get_user_info(state: Rc>) -> Result { - let api = { &state.borrow().borrow::().api.clone() }; + let mut state = Pin::new(state.borrow_mut()); + let api = state.borrow_mut::().api().await?; api.user_info().await.map_err(Error::from) } @@ -100,13 +143,9 @@ pub(crate) async fn get_access_token( ignore_certs: bool, ) -> Result { let refresh_token = get_refresh_token::call(state.clone())?; - let config = { - &state - .borrow() - .borrow::() - .config - .clone() - }; + let mut state = Pin::new(state.borrow_mut()); + let config = &state.borrow_mut::().await?.config; + let access_token = crate::auth::handle_refresh_tokens(&refresh_token, ignore_certs, &config.connection.uri) .await? @@ -118,13 +157,8 @@ pub(crate) async fn get_access_token( /// Equivalent to `phylum auth token`. #[op] pub(crate) fn get_refresh_token(state: Rc>) -> Result { - let config = { - &state - .borrow() - .borrow::() - .config - .clone() - }; + let mut state = state.borrow_mut(); + let config = &state.borrow_mut::().config; config .auth_info .offline_access @@ -139,7 +173,9 @@ pub(crate) async fn get_job_status( state: Rc>, job_id: Option<&str>, ) -> Result> { - let api = { &state.borrow().borrow::().api.clone() }; + let mut state = Pin::new(state.borrow_mut()); + let api = state.borrow_mut::().api().await?; + let job_id = job_id .map(|job_id| JobId::from_str(job_id).ok()) .unwrap_or_else(|| get_current_project().map(|p| p.id)) @@ -154,7 +190,9 @@ pub(crate) async fn get_project_details( state: Rc>, project_name: Option<&str>, ) -> Result { - let api = { &state.borrow().borrow::().api.clone() }; + let mut state = Pin::new(state.borrow_mut()); + let api = state.borrow_mut::().api().await?; + let project_name = project_name .map(String::from) .map(Result::Ok) @@ -177,7 +215,9 @@ pub(crate) async fn analyze_package( version: &str, package_type: &str, ) -> Result { - let api = { &state.borrow().borrow::().api.clone() }; + let mut state = Pin::new(state.borrow_mut()); + let api = state.borrow_mut::().api().await; + let package_type = PackageType::from_str(package_type) .map_err(|e| anyhow!("Unrecognized package type `{package_type}`: {e:?}"))?; api.get_package_details(&PackageDescriptor { @@ -203,3 +243,16 @@ pub(crate) fn parse_lockfile( parser.parse_file(Path::new(lockfile)) } + +pub(crate) fn api_decls() -> Vec { + vec![ + analyze::decl(), + get_user_info::decl(), + get_access_token::decl(), + get_refresh_token::decl(), + get_job_status::decl(), + get_project_details::decl(), + analyze_package::decl(), + parse_lockfile::decl(), + ] +} diff --git a/cli/src/commands/extensions/extension.rs b/cli/src/commands/extensions/extension.rs index 03ec11955..be5a5f48f 100644 --- a/cli/src/commands/extensions/extension.rs +++ b/cli/src/commands/extensions/extension.rs @@ -3,17 +3,23 @@ use std::fs::{self, DirBuilder}; #[cfg(unix)] use std::os::unix::fs::DirBuilderExt; use std::path::PathBuf; +use std::sync::Arc; use anyhow::{anyhow, Result}; +use futures::future::BoxFuture; use lazy_static::lazy_static; use regex::Regex; use serde::Deserialize; use walkdir::WalkDir; +use crate::api::PhylumApi; use crate::commands::{CommandResult, ExitCode}; +use crate::config::Config; use crate::deno::DenoRuntime; use crate::dirs; +use super::api::InjectedDependencies; + const MANIFEST_NAME: &str = "PhylumExt.toml"; lazy_static! { @@ -114,10 +120,16 @@ impl Extension { } /// Execute an extension subcommand. - pub async fn run(&self) -> CommandResult { + pub async fn run( + &self, + config: Config, + api: BoxFuture<'static, Result>>, + ) -> CommandResult { let script_path = self.path.join(&self.manifest.entry_point); - let mut deno = DenoRuntime::new(); + + let mut deno = DenoRuntime::new(InjectedDependencies::from_factories(api, config).await); deno.run(&script_path.to_string_lossy()).await?; + Ok(ExitCode::Ok.into()) } } diff --git a/cli/src/commands/extensions/mod.rs b/cli/src/commands/extensions/mod.rs index 0cdae2275..06337c1d3 100644 --- a/cli/src/commands/extensions/mod.rs +++ b/cli/src/commands/extensions/mod.rs @@ -1,13 +1,19 @@ mod api; mod extension; +pub(crate) use api::api_decls; +pub use extension::*; + +use std::sync::Arc; use std::{collections::HashSet, fs, io::ErrorKind, path::PathBuf}; +use crate::api::PhylumApi; use crate::commands::{CommandResult, CommandValue, ExitCode}; -pub use extension::*; +use crate::config::Config; use anyhow::{anyhow, Result}; use clap::{arg, ArgMatches, Command, ValueHint}; +use futures::future::BoxFuture; use log::{error, warn}; pub fn command<'a>() -> Command<'a> { @@ -72,6 +78,16 @@ pub async fn handle_extensions(matches: &ArgMatches) -> CommandResult { } } +/// Handle the `` command path. +/// Run the extension by name. +pub async fn handle_run_extension(name: &str, config: Config, api: BoxFuture<'static, Result>>) -> CommandResult { + let extension = Extension::load(name)?; + + extension.run(config, api).await?; + + Ok(CommandValue::Code(ExitCode::Ok)) +} + /// Handle the `extension add` subcommand path. /// Add the extension from the specified path. async fn handle_add_extension(path: &str) -> CommandResult { diff --git a/cli/src/config.rs b/cli/src/config.rs index 5ffee1770..5abef9d71 100644 --- a/cli/src/config.rs +++ b/cli/src/config.rs @@ -19,19 +19,19 @@ use crate::dirs; pub const PROJ_CONF_FILE: &str = ".phylum_project"; -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct ConnectionInfo { pub uri: String, } -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct AuthInfo { pub offline_access: Option, } pub type Packages = Vec; -#[derive(Debug, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct Config { pub connection: ConnectionInfo, pub auth_info: AuthInfo, diff --git a/cli/src/deno.rs b/cli/src/deno.rs index 07c9b46d8..faa9373e3 100644 --- a/cli/src/deno.rs +++ b/cli/src/deno.rs @@ -11,29 +11,31 @@ use deno_core::{ ModuleType, RuntimeOptions, }; +use crate::commands::extensions::api_decls; + /// Deno runtime state. pub struct DenoRuntime { runtime: JsRuntime, } -impl Default for DenoRuntime { - fn default() -> Self { - Self::new() - } -} - impl DenoRuntime { /// Create a new Deno runtime. - pub fn new() -> Self { + pub fn new(deps: T) -> Self { // TODO: Add Phylum API methods here. - let phylum_api = Extension::builder().build(); + let phylum_api = Extension::builder() + .ops(api_decls()) + .build(); - let runtime = JsRuntime::new(RuntimeOptions { + let mut runtime = JsRuntime::new(RuntimeOptions { module_loader: Some(Rc::new(TypescriptModuleLoader)), extensions: vec![phylum_api], ..Default::default() }); + let op_state = runtime.op_state(); + let mut op_state = op_state.borrow_mut(); + *op_state.borrow_mut() = deps; + Self { runtime } } diff --git a/cli/tests/extensions.rs b/cli/tests/extensions.rs index a009e8d82..778b5c8ed 100644 --- a/cli/tests/extensions.rs +++ b/cli/tests/extensions.rs @@ -261,6 +261,29 @@ fn conflicting_extension_name_is_filtered() { assert!(output.contains("extension was filtered out")); } +#[test] +fn api_extension() { + let tmp_dir = TmpDir::new(); + Command::cargo_bin("phylum") + .unwrap() + .env("XDG_DATA_HOME", &tmp_dir) + .arg("extension") + .arg("add") + .arg(fixtures_path().join("api-extension")) + .assert() + .success(); + + let cmd = Command::cargo_bin("phylum") + .unwrap() + .env("XDG_DATA_HOME", &tmp_dir) + .arg("api-extension") + .assert() + .success(); + + let output = std::str::from_utf8(&cmd.get_output().stderr).unwrap(); + assert!(output.contains("extension was filtered out")); +} + //////////////////////////////////////////////////////////////////////////////// // Utilities //////////////////////////////////////////////////////////////////////////////// diff --git a/cli/tests/fixtures/extensions/api-extension/PhylumExt.toml b/cli/tests/fixtures/extensions/api-extension/PhylumExt.toml new file mode 100644 index 000000000..7099031fe --- /dev/null +++ b/cli/tests/fixtures/extensions/api-extension/PhylumExt.toml @@ -0,0 +1,3 @@ +name = "api-extension" +description = "This extension tests the API ops" +entry_point = "main.ts" diff --git a/cli/tests/fixtures/extensions/api-extension/main.ts b/cli/tests/fixtures/extensions/api-extension/main.ts new file mode 100644 index 000000000..33d2a9db1 --- /dev/null +++ b/cli/tests/fixtures/extensions/api-extension/main.ts @@ -0,0 +1,3 @@ +Deno.core.print("Hello, World!\n"); + +console.log(await Deno.core.opAsync("get_user_info")); From 00033fa311e37c2cbe4c899aec3919f4ed165597 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Thu, 2 Jun 2022 18:21:21 +0200 Subject: [PATCH 35/52] PhylumApi owns Config --- cli/src/api/mod.rs | 105 ++++++++++++++++------- cli/src/bin/phylum.rs | 24 +++--- cli/src/commands/auth.rs | 5 +- cli/src/commands/extensions/api.rs | 72 +++++++--------- cli/src/commands/extensions/extension.rs | 9 +- cli/src/commands/extensions/mod.rs | 9 +- cli/src/commands/jobs.rs | 5 +- cli/src/commands/packages.rs | 3 +- cli/src/test.rs | 12 ++- 9 files changed, 137 insertions(+), 107 deletions(-) diff --git a/cli/src/api/mod.rs b/cli/src/api/mod.rs index 3942e61cf..6180fb67a 100644 --- a/cli/src/api/mod.rs +++ b/cli/src/api/mod.rs @@ -33,7 +33,7 @@ type Result = std::result::Result; pub struct PhylumApi { config: Config, client: Client, - api_uri: String, + //api_uri: String, ignore_certs: bool, } @@ -104,20 +104,21 @@ impl PhylumApi { /// must be obtained, the auth_info struct will be updated with the new /// information. It is the duty of the calling code to save any changes pub async fn new( - auth_info: &mut AuthInfo, - api_uri: &str, + mut config: Config, request_timeout: Option, ignore_certs: bool, ) -> Result { // Do we have a refresh token? - let tokens: TokenResponse = match &auth_info.offline_access { + let tokens: TokenResponse = match &config.auth_info.offline_access { Some(refresh_token) => { - handle_refresh_tokens(refresh_token, ignore_certs, api_uri).await? + handle_refresh_tokens(refresh_token, ignore_certs, &config.connection.uri).await? + } + None => { + handle_auth_flow(&AuthAction::Login, ignore_certs, &config.connection.uri).await? } - None => handle_auth_flow(&AuthAction::Login, ignore_certs, api_uri).await?, }; - auth_info.offline_access = Some(tokens.refresh_token.clone()); + config.auth_info.offline_access = Some(tokens.refresh_token.clone()); let version = env!("CARGO_PKG_VERSION"); let mut headers = HeaderMap::new(); @@ -140,8 +141,8 @@ impl PhylumApi { .build()?; Ok(Self { + config, client, - api_uri: api_uri.to_string(), ignore_certs, }) } @@ -175,14 +176,15 @@ impl PhylumApi { /// Ping the system and verify it's up pub async fn ping(&self) -> Result { Ok(self - .get::(endpoints::get_ping(&self.api_uri)) + .get::(endpoints::get_ping(&self.config.connection.uri)) .await? .response) } /// Get information about the authenticated user pub async fn user_info(&self) -> Result { - let oidc_settings = fetch_oidc_server_settings(self.ignore_certs, &self.api_uri).await?; + let oidc_settings = + fetch_oidc_server_settings(self.ignore_certs, &self.config.connection.uri).await?; self.get(oidc_settings.userinfo_endpoint.into()).await } @@ -190,7 +192,7 @@ impl PhylumApi { pub async fn create_project(&self, name: &str, group: Option<&str>) -> Result { Ok(self .put::( - endpoints::put_create_project(&self.api_uri), + endpoints::put_create_project(&self.config.connection.uri), CreateProjectRequest { name: name.to_owned(), group_name: group.map(String::from), @@ -203,8 +205,8 @@ impl PhylumApi { /// Get a list of projects pub async fn get_projects(&self, group: Option<&str>) -> Result> { let uri = match group { - Some(group) => endpoints::group_project_summary(&self.api_uri, group), - None => endpoints::get_project_summary(&self.api_uri), + Some(group) => endpoints::group_project_summary(&self.config.connection.uri, group), + None => endpoints::get_project_summary(&self.config.connection.uri), }; self.get(uri).await @@ -212,13 +214,17 @@ impl PhylumApi { /// Get user settings pub async fn get_user_settings(&self) -> Result { - self.get(endpoints::get_user_settings(&self.api_uri)).await + self.get(endpoints::get_user_settings(&self.config.connection.uri)) + .await } /// Put updated user settings pub async fn put_user_settings(&self, settings: &UserSettings) -> Result { - self.put::(endpoints::put_user_settings(&self.api_uri), &settings) - .await?; + self.put::( + endpoints::put_user_settings(&self.config.connection.uri), + &settings, + ) + .await?; Ok(true) } @@ -242,15 +248,22 @@ impl PhylumApi { }; log::debug!("==> Sending package submission: {:?}", req); let resp: SubmitPackageResponse = self - .put(endpoints::put_submit_package(&self.api_uri), req) + .put( + endpoints::put_submit_package(&self.config.connection.uri), + req, + ) .await?; Ok(resp.job_id) } /// Get the status of a previously submitted job pub async fn get_job_status(&self, job_id: &JobId) -> Result> { - self.get(endpoints::get_job_status(&self.api_uri, job_id, false)) - .await + self.get(endpoints::get_job_status( + &self.config.connection.uri, + job_id, + false, + )) + .await } /// Get the status of a previously submitted job (verbose output) @@ -258,20 +271,30 @@ impl PhylumApi { &self, job_id: &JobId, ) -> Result> { - self.get(endpoints::get_job_status(&self.api_uri, job_id, true)) - .await + self.get(endpoints::get_job_status( + &self.config.connection.uri, + job_id, + true, + )) + .await } /// Get the status of all jobs pub async fn get_status(&self) -> Result { - self.get(endpoints::get_all_jobs_status(&self.api_uri, 30)) - .await + self.get(endpoints::get_all_jobs_status( + &self.config.connection.uri, + 30, + )) + .await } /// Get the details of a specific project pub async fn get_project_details(&self, project_name: &str) -> Result { - self.get(endpoints::get_project_details(&self.api_uri, project_name)) - .await + self.get(endpoints::get_project_details( + &self.config.connection.uri, + project_name, + )) + .await } /// Resolve a Project Name to a Project ID @@ -297,13 +320,17 @@ impl PhylumApi { /// Get package details pub async fn get_package_details(&self, pkg: &PackageDescriptor) -> Result { - self.get(endpoints::get_package_status(&self.api_uri, pkg)) - .await + self.get(endpoints::get_package_status( + &self.config.connection.uri, + pkg, + )) + .await } /// Get all groups the user is part of. pub async fn get_groups_list(&self) -> Result { - self.get(endpoints::group_list(&self.api_uri)).await + self.get(endpoints::group_list(&self.config.connection.uri)) + .await } /// Get all groups the user is part of. @@ -311,9 +338,13 @@ impl PhylumApi { let group = CreateGroupRequest { group_name: group_name.into(), }; - self.post(endpoints::group_create(&self.api_uri), group) + self.post(endpoints::group_create(&self.config.connection.uri), group) .await } + + pub fn config(&self) -> &Config { + &self.config + } } /// Tests @@ -326,6 +357,7 @@ mod tests { use wiremock::matchers::{method, path, path_regex, query_param}; use wiremock::{Mock, ResponseTemplate}; + use crate::config::ConnectionInfo; use crate::test::mockito::*; use super::*; @@ -339,11 +371,20 @@ mod tests { #[tokio::test] async fn when_creating_unauthenticated_phylum_api_it_auths_itself() -> Result<()> { let mock_server = build_mock_server().await; - let mut auth_info = build_unauthenticated_auth_info(); - PhylumApi::new(&mut auth_info, mock_server.uri().as_str(), None, false).await?; + let auth_info = build_unauthenticated_auth_info(); + + let config = Config { + connection: ConnectionInfo { + uri: mock_server.uri(), + }, + auth_info, + ..Default::default() + }; + + let api = PhylumApi::new(config, None, false).await?; // After auth, auth_info should have a offline access token assert!( - auth_info.offline_access.is_some(), + api.config().auth_info.offline_access.is_some(), "Offline access token was not set" ); diff --git a/cli/src/bin/phylum.rs b/cli/src/bin/phylum.rs index 86ac5d53a..dbe212224 100644 --- a/cli/src/bin/phylum.rs +++ b/cli/src/bin/phylum.rs @@ -1,5 +1,4 @@ -use std::path::{Path, PathBuf}; -use std::sync::Arc; +use std::path::PathBuf; use std::time::UNIX_EPOCH; use anyhow::{anyhow, Context, Result}; @@ -51,14 +50,14 @@ pub fn exit_error(error: Box, message: impl AsRef) - /// Construct an instance of `PhylumApi` given configuration, optional timeout, and whether we /// need API to ignore certificates. async fn api_factory( - config: &mut Config, - config_path: &Path, + config: Config, + config_path: PathBuf, timeout: Option, ignore_certs: bool, ) -> Result { let api = PhylumApi::new( - &mut config.auth_info, - &config.connection.uri, + // &mut config.auth_info, + config, timeout, ignore_certs, ) @@ -66,7 +65,7 @@ async fn api_factory( .context("Error creating client")?; // PhylumApi may have had to log in, updating the auth info so we should save the config - save_config(config_path, &config).with_context(|| { + save_config(&config_path, api.config()).with_context(|| { format!( "Failed to save configuration to '{}'", config_path.to_string_lossy() @@ -162,7 +161,7 @@ async fn handle_commands() -> CommandResult { // Get the future, but don't await. Commands that require access to the API will await on this, // so that the API is not instantiated ahead of time for subcommands that don't require it. - let api = api_factory(&mut config, &config_path, timeout, ignore_certs); + let api = api_factory(config.clone(), config_path.clone(), timeout, ignore_certs); let (subcommand, matches) = matches.subcommand().unwrap(); match subcommand { @@ -183,10 +182,10 @@ async fn handle_commands() -> CommandResult { "parse" => handle_parse(matches), "ping" => handle_ping(api.await?).await, "project" => handle_project(&mut api.await?, matches).await, - "package" => handle_get_package(&mut api.await?, &config.request_type, matches).await, - "history" => handle_submission(&mut api.await?, config, matches).await, + "package" => handle_get_package(&mut api.await?, matches).await, + "history" => handle_submission(&mut api.await?, matches).await, "group" => handle_group(&mut api.await?, matches).await, - "analyze" | "batch" => handle_submission(&mut api.await?, config, matches).await, + "analyze" | "batch" => handle_submission(&mut api.await?, matches).await, #[cfg(feature = "selfmanage")] "uninstall" => handle_uninstall(matches), @@ -198,8 +197,7 @@ async fn handle_commands() -> CommandResult { extension_subcmd => { handle_run_extension( extension_subcmd, - config.clone(), - Box::pin(async { api.await.map(Arc::new) }), + Box::pin(api), ) .await } // Extension::load(extension_subcmd)?.run().await, diff --git a/cli/src/commands/auth.rs b/cli/src/commands/auth.rs index e8ef048a5..a9a572816 100644 --- a/cli/src/commands/auth.rs +++ b/cli/src/commands/auth.rs @@ -39,7 +39,7 @@ async fn handle_auth_login( /// Display the current authentication status to the user. pub async fn handle_auth_status( - mut config: Config, + config: Config, timeout: Option, ignore_certs: bool, ) -> CommandResult { @@ -50,8 +50,7 @@ pub async fn handle_auth_status( // Create a client with our auth token attached. let api = PhylumApi::new( - &mut config.auth_info, - &config.connection.uri, + config, timeout, ignore_certs, ) diff --git a/cli/src/commands/extensions/api.rs b/cli/src/commands/extensions/api.rs index 5e748f5e5..817c40e06 100644 --- a/cli/src/commands/extensions/api.rs +++ b/cli/src/commands/extensions/api.rs @@ -3,7 +3,7 @@ //! implementation, having these functions unused would break CI. #![allow(unused)] -use std::cell::RefCell; +use std::cell::{RefCell, RefMut}; use std::future::Future; use std::path::Path; use std::pin::Pin; @@ -31,8 +31,8 @@ use phylum_types::types::project::ProjectDetailsResponse; // BROKEN -enum OnceFuture { - Future(Option>>), +pub(crate) enum OnceFuture { + Future(Option>), Awaited(T), } @@ -41,51 +41,39 @@ impl OnceFuture { OnceFuture::Future(Some(t)) } - async fn try_get(&mut self) -> Result<&T> { + async fn get(&mut self) -> &T { match *self { - OnceFuture::Future(Some(ref mut t)) => { - *self = OnceFuture::Awaited(t.await?); + OnceFuture::Future(ref mut t) => { + *self = OnceFuture::Awaited(t.take().unwrap().await); match *self { OnceFuture::Future(..) => unreachable!(), - OnceFuture::Awaited(ref mut t) => Ok(t), + OnceFuture::Awaited(ref mut t) => t, } } - OnceFuture::Awaited(ref mut t) => Ok(t) + OnceFuture::Awaited(ref mut t) => t, } } } -type PhylumApiFut = OnceFuture>; +pub(crate) type ExtensionState = OnceFuture>; -/// Container for lazily evaluated dependencies of Extensions API functions. These values won't be -/// visible from extension code and will have to be set up by the JS runtime builder which will -/// load their configuration as any other command and then provide the pertinent factories. -pub(crate) struct InjectedDependencies { - api: OnceFuture>>, - config: Config, -} - -impl InjectedDependencies { - pub(crate) async fn from_factories( - api_factory: BoxFuture<'static, Result>>, - config: Config, - ) -> Self { - InjectedDependencies { - api: OnceFuture::new(api_factory), - config, - } - } - - async fn api(&mut self) -> Result> { - self.api +impl ExtensionState { + async fn borrow_from(state: &mut OpState) -> Result<&PhylumApi> { + state + .borrow_mut::() .get() .await .as_ref() - .cloned() .map_err(|e| anyhow!("{:?}", e)) } } +impl From>> for ExtensionState { + fn from(t: BoxFuture<'static, Result>) -> Self { + OnceFuture::new(t) + } +} + /// Analyze a lockfile. /// Equivalent to `phylum analyze`. #[op] @@ -96,7 +84,7 @@ pub(crate) async fn analyze( group: Option<&str>, ) -> Result { let mut state = Pin::new(state.borrow_mut()); - let api = state.borrow_mut::().get().await?; + let api = ExtensionState::borrow_from(&mut state).await?; let (packages, request_type) = get_packages_from_lockfile(Path::new(lockfile)) .context("Unable to locate any valid package in package lockfile")?; @@ -131,7 +119,8 @@ pub(crate) async fn analyze( #[op] pub(crate) async fn get_user_info(state: Rc>) -> Result { let mut state = Pin::new(state.borrow_mut()); - let api = state.borrow_mut::().api().await?; + let api = ExtensionState::borrow_from(&mut state).await?; + api.user_info().await.map_err(Error::from) } @@ -142,9 +131,9 @@ pub(crate) async fn get_access_token( state: Rc>, ignore_certs: bool, ) -> Result { - let refresh_token = get_refresh_token::call(state.clone())?; + let refresh_token = get_refresh_token::call(state.clone()).await?; let mut state = Pin::new(state.borrow_mut()); - let config = &state.borrow_mut::().await?.config; + let config = ExtensionState::borrow_from(&mut state).await?.config(); let access_token = crate::auth::handle_refresh_tokens(&refresh_token, ignore_certs, &config.connection.uri) @@ -156,9 +145,10 @@ pub(crate) async fn get_access_token( /// Retrieve the refresh token. /// Equivalent to `phylum auth token`. #[op] -pub(crate) fn get_refresh_token(state: Rc>) -> Result { - let mut state = state.borrow_mut(); - let config = &state.borrow_mut::().config; +pub(crate) async fn get_refresh_token(state: Rc>) -> Result { + let mut state = Pin::new(state.borrow_mut()); + let config = ExtensionState::borrow_from(&mut state).await?.config(); + config .auth_info .offline_access @@ -174,7 +164,7 @@ pub(crate) async fn get_job_status( job_id: Option<&str>, ) -> Result> { let mut state = Pin::new(state.borrow_mut()); - let api = state.borrow_mut::().api().await?; + let api = ExtensionState::borrow_from(&mut state).await?; let job_id = job_id .map(|job_id| JobId::from_str(job_id).ok()) @@ -191,7 +181,7 @@ pub(crate) async fn get_project_details( project_name: Option<&str>, ) -> Result { let mut state = Pin::new(state.borrow_mut()); - let api = state.borrow_mut::().api().await?; + let api = ExtensionState::borrow_from(&mut state).await?; let project_name = project_name .map(String::from) @@ -216,7 +206,7 @@ pub(crate) async fn analyze_package( package_type: &str, ) -> Result { let mut state = Pin::new(state.borrow_mut()); - let api = state.borrow_mut::().api().await; + let api = ExtensionState::borrow_from(&mut state).await?; let package_type = PackageType::from_str(package_type) .map_err(|e| anyhow!("Unrecognized package type `{package_type}`: {e:?}"))?; diff --git a/cli/src/commands/extensions/extension.rs b/cli/src/commands/extensions/extension.rs index be5a5f48f..77f6283ae 100644 --- a/cli/src/commands/extensions/extension.rs +++ b/cli/src/commands/extensions/extension.rs @@ -3,7 +3,6 @@ use std::fs::{self, DirBuilder}; #[cfg(unix)] use std::os::unix::fs::DirBuilderExt; use std::path::PathBuf; -use std::sync::Arc; use anyhow::{anyhow, Result}; use futures::future::BoxFuture; @@ -14,11 +13,10 @@ use walkdir::WalkDir; use crate::api::PhylumApi; use crate::commands::{CommandResult, ExitCode}; -use crate::config::Config; use crate::deno::DenoRuntime; use crate::dirs; -use super::api::InjectedDependencies; +use super::api::ExtensionState; const MANIFEST_NAME: &str = "PhylumExt.toml"; @@ -122,12 +120,11 @@ impl Extension { /// Execute an extension subcommand. pub async fn run( &self, - config: Config, - api: BoxFuture<'static, Result>>, + api: BoxFuture<'static, Result>, ) -> CommandResult { let script_path = self.path.join(&self.manifest.entry_point); - let mut deno = DenoRuntime::new(InjectedDependencies::from_factories(api, config).await); + let mut deno = DenoRuntime::new(ExtensionState::from(api)); deno.run(&script_path.to_string_lossy()).await?; Ok(ExitCode::Ok.into()) diff --git a/cli/src/commands/extensions/mod.rs b/cli/src/commands/extensions/mod.rs index 06337c1d3..53f824e1d 100644 --- a/cli/src/commands/extensions/mod.rs +++ b/cli/src/commands/extensions/mod.rs @@ -4,12 +4,10 @@ mod extension; pub(crate) use api::api_decls; pub use extension::*; -use std::sync::Arc; use std::{collections::HashSet, fs, io::ErrorKind, path::PathBuf}; use crate::api::PhylumApi; use crate::commands::{CommandResult, CommandValue, ExitCode}; -use crate::config::Config; use anyhow::{anyhow, Result}; use clap::{arg, ArgMatches, Command, ValueHint}; @@ -80,10 +78,13 @@ pub async fn handle_extensions(matches: &ArgMatches) -> CommandResult { /// Handle the `` command path. /// Run the extension by name. -pub async fn handle_run_extension(name: &str, config: Config, api: BoxFuture<'static, Result>>) -> CommandResult { +pub async fn handle_run_extension( + name: &str, + api: BoxFuture<'static, Result>, +) -> CommandResult { let extension = Extension::load(name)?; - extension.run(config, api).await?; + extension.run(api).await?; Ok(CommandValue::Code(ExitCode::Ok)) } diff --git a/cli/src/commands/jobs.rs b/cli/src/commands/jobs.rs index f33114538..b1620307a 100644 --- a/cli/src/commands/jobs.rs +++ b/cli/src/commands/jobs.rs @@ -15,7 +15,7 @@ use phylum_types::types::package::*; use crate::api::{PhylumApi, PhylumApiError}; use crate::commands::parse::get_packages_from_lockfile; use crate::commands::{CommandResult, CommandValue}; -use crate::config::{get_current_project, Config, ProjectConfig}; +use crate::config::{get_current_project, ProjectConfig}; use crate::filter::Filter; use crate::print::print_response; use crate::print_user_success; @@ -155,11 +155,10 @@ pub async fn handle_history(api: &mut PhylumApi, matches: &clap::ArgMatches) -> /// displays summary information about the submitted package(s) pub async fn handle_submission( api: &mut PhylumApi, - config: Config, matches: &clap::ArgMatches, ) -> CommandResult { let mut packages = vec![]; - let mut request_type = config.request_type; // default request type + let mut request_type = api.config().request_type; // default request type let mut synch = false; // get status after submission let mut verbose = false; let mut pretty_print = false; diff --git a/cli/src/commands/packages.rs b/cli/src/commands/packages.rs index 25cc80205..171e8a600 100644 --- a/cli/src/commands/packages.rs +++ b/cli/src/commands/packages.rs @@ -37,11 +37,10 @@ fn parse_package(options: &ArgMatches, request_type: &PackageType) -> Option CommandResult { let pretty_print = !matches.is_present("json"); - let pkg = parse_package(matches, req_type); + let pkg = parse_package(matches, &api.config().request_type); if pkg.is_none() { return Err(anyhow!("Could not find or parse package information")); } diff --git a/cli/src/test.rs b/cli/src/test.rs index 1048e58c9..0a1b15621 100644 --- a/cli/src/test.rs +++ b/cli/src/test.rs @@ -24,7 +24,7 @@ pub mod mockito { use crate::api::{PhylumApi, PhylumApiError}; use crate::auth::OidcServerSettings; - use crate::config::AuthInfo; + use crate::config::{AuthInfo, Config, ConnectionInfo}; use phylum_types::types::auth::*; pub const DUMMY_REFRESH_TOKEN: &str = "DUMMY_REFRESH_TOKEN"; @@ -139,9 +139,15 @@ pub mod mockito { } pub async fn build_phylum_api(mock_server: &MockServer) -> Result { + let config = Config { + connection: ConnectionInfo { + uri: mock_server.uri() + }, + auth_info: build_authenticated_auth_info(), + ..Default::default() + }; let phylum = PhylumApi::new( - &mut build_authenticated_auth_info(), - mock_server.uri().as_str(), + config, None, false, ) From 1d7021e588650518fb7c518b6bda6d35c58e078c Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Thu, 2 Jun 2022 18:29:01 +0200 Subject: [PATCH 36/52] Cargo fmt --- cli/src/bin/phylum.rs | 8 +------- cli/src/commands/auth.rs | 10 +++------- cli/src/commands/extensions/extension.rs | 5 +---- cli/src/commands/jobs.rs | 5 +---- cli/src/commands/packages.rs | 5 +---- cli/src/deno.rs | 4 +--- cli/src/test.rs | 9 ++------- 7 files changed, 10 insertions(+), 36 deletions(-) diff --git a/cli/src/bin/phylum.rs b/cli/src/bin/phylum.rs index dbe212224..f8343491c 100644 --- a/cli/src/bin/phylum.rs +++ b/cli/src/bin/phylum.rs @@ -194,13 +194,7 @@ async fn handle_commands() -> CommandResult { "extension" => handle_extensions(matches).await, #[cfg(feature = "extensions")] - extension_subcmd => { - handle_run_extension( - extension_subcmd, - Box::pin(api), - ) - .await - } // Extension::load(extension_subcmd)?.run().await, + extension_subcmd => handle_run_extension(extension_subcmd, Box::pin(api)).await, // Extension::load(extension_subcmd)?.run().await, #[cfg(not(feature = "extensions"))] _ => unreachable!(), diff --git a/cli/src/commands/auth.rs b/cli/src/commands/auth.rs index a9a572816..0bd786fdf 100644 --- a/cli/src/commands/auth.rs +++ b/cli/src/commands/auth.rs @@ -49,13 +49,9 @@ pub async fn handle_auth_status( } // Create a client with our auth token attached. - let api = PhylumApi::new( - config, - timeout, - ignore_certs, - ) - .await - .context("Error creating client")?; + let api = PhylumApi::new(config, timeout, ignore_certs) + .await + .context("Error creating client")?; let user_info = api.user_info().await; diff --git a/cli/src/commands/extensions/extension.rs b/cli/src/commands/extensions/extension.rs index 77f6283ae..5d899f190 100644 --- a/cli/src/commands/extensions/extension.rs +++ b/cli/src/commands/extensions/extension.rs @@ -118,10 +118,7 @@ impl Extension { } /// Execute an extension subcommand. - pub async fn run( - &self, - api: BoxFuture<'static, Result>, - ) -> CommandResult { + pub async fn run(&self, api: BoxFuture<'static, Result>) -> CommandResult { let script_path = self.path.join(&self.manifest.entry_point); let mut deno = DenoRuntime::new(ExtensionState::from(api)); diff --git a/cli/src/commands/jobs.rs b/cli/src/commands/jobs.rs index b1620307a..5dc54c3cd 100644 --- a/cli/src/commands/jobs.rs +++ b/cli/src/commands/jobs.rs @@ -153,10 +153,7 @@ pub async fn handle_history(api: &mut PhylumApi, matches: &clap::ArgMatches) -> /// Handles submission of packages to the system for analysis and /// displays summary information about the submitted package(s) -pub async fn handle_submission( - api: &mut PhylumApi, - matches: &clap::ArgMatches, -) -> CommandResult { +pub async fn handle_submission(api: &mut PhylumApi, matches: &clap::ArgMatches) -> CommandResult { let mut packages = vec![]; let mut request_type = api.config().request_type; // default request type let mut synch = false; // get status after submission diff --git a/cli/src/commands/packages.rs b/cli/src/commands/packages.rs index 171e8a600..adb54124a 100644 --- a/cli/src/commands/packages.rs +++ b/cli/src/commands/packages.rs @@ -35,10 +35,7 @@ fn parse_package(options: &ArgMatches, request_type: &PackageType) -> Option CommandResult { +pub async fn handle_get_package(api: &mut PhylumApi, matches: &clap::ArgMatches) -> CommandResult { let pretty_print = !matches.is_present("json"); let pkg = parse_package(matches, &api.config().request_type); if pkg.is_none() { diff --git a/cli/src/deno.rs b/cli/src/deno.rs index faa9373e3..92de1dd77 100644 --- a/cli/src/deno.rs +++ b/cli/src/deno.rs @@ -22,9 +22,7 @@ impl DenoRuntime { /// Create a new Deno runtime. pub fn new(deps: T) -> Self { // TODO: Add Phylum API methods here. - let phylum_api = Extension::builder() - .ops(api_decls()) - .build(); + let phylum_api = Extension::builder().ops(api_decls()).build(); let mut runtime = JsRuntime::new(RuntimeOptions { module_loader: Some(Rc::new(TypescriptModuleLoader)), diff --git a/cli/src/test.rs b/cli/src/test.rs index 0a1b15621..ecb05cdfb 100644 --- a/cli/src/test.rs +++ b/cli/src/test.rs @@ -141,17 +141,12 @@ pub mod mockito { pub async fn build_phylum_api(mock_server: &MockServer) -> Result { let config = Config { connection: ConnectionInfo { - uri: mock_server.uri() + uri: mock_server.uri(), }, auth_info: build_authenticated_auth_info(), ..Default::default() }; - let phylum = PhylumApi::new( - config, - None, - false, - ) - .await?; + let phylum = PhylumApi::new(config, None, false).await?; Ok(phylum) } } From 63e2f98727e34e771c926105a30cf43bbede5fd8 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Mon, 6 Jun 2022 15:33:33 +0200 Subject: [PATCH 37/52] Extension state newtype --- cli/src/commands/extensions/api.rs | 5 +++-- cli/src/commands/extensions/extension.rs | 2 +- cli/src/deno.rs | 11 ++++++----- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/cli/src/commands/extensions/api.rs b/cli/src/commands/extensions/api.rs index 817c40e06..77ea4ea1d 100644 --- a/cli/src/commands/extensions/api.rs +++ b/cli/src/commands/extensions/api.rs @@ -55,12 +55,13 @@ impl OnceFuture { } } -pub(crate) type ExtensionState = OnceFuture>; +pub struct ExtensionState(OnceFuture>); impl ExtensionState { async fn borrow_from(state: &mut OpState) -> Result<&PhylumApi> { state .borrow_mut::() + .0 .get() .await .as_ref() @@ -70,7 +71,7 @@ impl ExtensionState { impl From>> for ExtensionState { fn from(t: BoxFuture<'static, Result>) -> Self { - OnceFuture::new(t) + Self(OnceFuture::new(t)) } } diff --git a/cli/src/commands/extensions/extension.rs b/cli/src/commands/extensions/extension.rs index 5d899f190..d42dee37a 100644 --- a/cli/src/commands/extensions/extension.rs +++ b/cli/src/commands/extensions/extension.rs @@ -16,7 +16,7 @@ use crate::commands::{CommandResult, ExitCode}; use crate::deno::DenoRuntime; use crate::dirs; -use super::api::ExtensionState; +pub(crate) use super::api::ExtensionState; const MANIFEST_NAME: &str = "PhylumExt.toml"; diff --git a/cli/src/deno.rs b/cli/src/deno.rs index 92de1dd77..73bc2341f 100644 --- a/cli/src/deno.rs +++ b/cli/src/deno.rs @@ -11,7 +11,7 @@ use deno_core::{ ModuleType, RuntimeOptions, }; -use crate::commands::extensions::api_decls; +use crate::commands::extensions::{api_decls, ExtensionState}; /// Deno runtime state. pub struct DenoRuntime { @@ -20,9 +20,10 @@ pub struct DenoRuntime { impl DenoRuntime { /// Create a new Deno runtime. - pub fn new(deps: T) -> Self { - // TODO: Add Phylum API methods here. - let phylum_api = Extension::builder().ops(api_decls()).build(); + pub fn new(deps: ExtensionState) -> Self { + let phylum_api = Extension::builder() + .ops(api_decls()) + .build(); let mut runtime = JsRuntime::new(RuntimeOptions { module_loader: Some(Rc::new(TypescriptModuleLoader)), @@ -32,7 +33,7 @@ impl DenoRuntime { let op_state = runtime.op_state(); let mut op_state = op_state.borrow_mut(); - *op_state.borrow_mut() = deps; + op_state.put(deps); Self { runtime } } From 77581854a15185426645a45032cde50914cb3399 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Mon, 6 Jun 2022 16:52:53 +0200 Subject: [PATCH 38/52] Op wrappers in fixture --- cli/tests/extensions.rs | 5 ++- .../fixtures/extensions/api-extension/main.ts | 32 +++++++++++++++++-- 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/cli/tests/extensions.rs b/cli/tests/extensions.rs index 778b5c8ed..8ba746610 100644 --- a/cli/tests/extensions.rs +++ b/cli/tests/extensions.rs @@ -280,8 +280,11 @@ fn api_extension() { .assert() .success(); + let output = std::str::from_utf8(&cmd.get_output().stdout).unwrap(); + println!("{}", output); let output = std::str::from_utf8(&cmd.get_output().stderr).unwrap(); - assert!(output.contains("extension was filtered out")); + println!("{}", output); + // assert!(output.contains("extension was filtered out")); } //////////////////////////////////////////////////////////////////////////////// diff --git a/cli/tests/fixtures/extensions/api-extension/main.ts b/cli/tests/fixtures/extensions/api-extension/main.ts index 33d2a9db1..7ca752543 100644 --- a/cli/tests/fixtures/extensions/api-extension/main.ts +++ b/cli/tests/fixtures/extensions/api-extension/main.ts @@ -1,3 +1,31 @@ -Deno.core.print("Hello, World!\n"); +async function analyze(lockfile: string, project?: string, group?: string) { + return await Deno.core.opAsync("analyze", lockfile, project, group) +} -console.log(await Deno.core.opAsync("get_user_info")); +async function get_user_info() { + return await Deno.core.opAsync("get_user_info") +} + +async function get_access_token(ignore_certs?: bool) { + return await Deno.core.opAsync("get_access_token" ?? false) +} + +async function get_refresh_token() { + return await Deno.core.opAsync("get_refresh_token") +} + +async function get_job_status(job_id?: string) { + return await Deno.core.opAsync("get_job_status", job_id) +} + +async function get_project_details(project_name?: string) { + return await Deno.core.opAsync("get_project_details", project_name) +} + +async function analyze_package(name: string, version: string, ecosystem: string) { + return await Deno.core.opAsync("analyze_package", name, version, ecosystem) +} + +async function parse_lockfile(lockfile: string, lockfile_type: string) { + return await Deno.core.opAsync("parse_lockfile", lockfile, lockfile_type) +} From 54c3652e5e750d9538519f44b1d6edb0046a4801 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Mon, 6 Jun 2022 19:18:06 +0200 Subject: [PATCH 39/52] Separated wrapper module, op calls tests --- cli/src/deno.rs | 4 +- cli/tests/extensions.rs | 12 +++-- .../fixtures/extensions/api-extension/main.ts | 44 ++++++------------- .../extensions/api-extension/phylum-api.ts | 31 +++++++++++++ 4 files changed, 54 insertions(+), 37 deletions(-) create mode 100644 cli/tests/fixtures/extensions/api-extension/phylum-api.ts diff --git a/cli/src/deno.rs b/cli/src/deno.rs index 73bc2341f..65a28b1e2 100644 --- a/cli/src/deno.rs +++ b/cli/src/deno.rs @@ -21,9 +21,7 @@ pub struct DenoRuntime { impl DenoRuntime { /// Create a new Deno runtime. pub fn new(deps: ExtensionState) -> Self { - let phylum_api = Extension::builder() - .ops(api_decls()) - .build(); + let phylum_api = Extension::builder().ops(api_decls()).build(); let mut runtime = JsRuntime::new(RuntimeOptions { module_loader: Some(Rc::new(TypescriptModuleLoader)), diff --git a/cli/tests/extensions.rs b/cli/tests/extensions.rs index 8ba746610..67a00d5c0 100644 --- a/cli/tests/extensions.rs +++ b/cli/tests/extensions.rs @@ -280,10 +280,14 @@ fn api_extension() { .assert() .success(); - let output = std::str::from_utf8(&cmd.get_output().stdout).unwrap(); - println!("{}", output); - let output = std::str::from_utf8(&cmd.get_output().stderr).unwrap(); - println!("{}", output); + { + let output = std::str::from_utf8(&cmd.get_output().stdout).unwrap(); + println!("{}", output); + } + { + let output = std::str::from_utf8(&cmd.get_output().stderr).unwrap(); + println!("{}", output); + } // assert!(output.contains("extension was filtered out")); } diff --git a/cli/tests/fixtures/extensions/api-extension/main.ts b/cli/tests/fixtures/extensions/api-extension/main.ts index 7ca752543..2746832a1 100644 --- a/cli/tests/fixtures/extensions/api-extension/main.ts +++ b/cli/tests/fixtures/extensions/api-extension/main.ts @@ -1,31 +1,15 @@ -async function analyze(lockfile: string, project?: string, group?: string) { - return await Deno.core.opAsync("analyze", lockfile, project, group) -} - -async function get_user_info() { - return await Deno.core.opAsync("get_user_info") -} - -async function get_access_token(ignore_certs?: bool) { - return await Deno.core.opAsync("get_access_token" ?? false) -} - -async function get_refresh_token() { - return await Deno.core.opAsync("get_refresh_token") -} - -async function get_job_status(job_id?: string) { - return await Deno.core.opAsync("get_job_status", job_id) -} - -async function get_project_details(project_name?: string) { - return await Deno.core.opAsync("get_project_details", project_name) -} - -async function analyze_package(name: string, version: string, ecosystem: string) { - return await Deno.core.opAsync("analyze_package", name, version, ecosystem) -} - -async function parse_lockfile(lockfile: string, lockfile_type: string) { - return await Deno.core.opAsync("parse_lockfile", lockfile, lockfile_type) +import * as PhylumApi from './phylum-api.ts' + +Deno.core.print(JSON.stringify(await PhylumApi.get_user_info())) +await PhylumApi.get_access_token() +await PhylumApi.get_refresh_token() +try { + await PhylumApi.get_job_status() +} catch(e) { + Deno.core.print(JSON.stringify(e)) +} +try { + await PhylumApi.get_project_details() +} catch(e) { + Deno.core.print(JSON.stringify(e)) } diff --git a/cli/tests/fixtures/extensions/api-extension/phylum-api.ts b/cli/tests/fixtures/extensions/api-extension/phylum-api.ts new file mode 100644 index 000000000..3df11be4b --- /dev/null +++ b/cli/tests/fixtures/extensions/api-extension/phylum-api.ts @@ -0,0 +1,31 @@ +export async function analyze(lockfile: string, project?: string, group?: string) { + return await Deno.core.opAsync("analyze", lockfile, project, group) +} + +export async function get_user_info() { + return await Deno.core.opAsync("get_user_info") +} + +export async function get_access_token(ignore_certs?: bool) { + return await Deno.core.opAsync("get_access_token" ?? false) +} + +export async function get_refresh_token() { + return await Deno.core.opAsync("get_refresh_token") +} + +export async function get_job_status(job_id?: string) { + return await Deno.core.opAsync("get_job_status", job_id) +} + +export async function get_project_details(project_name?: string) { + return await Deno.core.opAsync("get_project_details", project_name) +} + +export async function analyze_package(name: string, version: string, ecosystem: string) { + return await Deno.core.opAsync("analyze_package", name, version, ecosystem) +} + +export async function parse_lockfile(lockfile: string, lockfile_type: string) { + return await Deno.core.opAsync("parse_lockfile", lockfile, lockfile_type) +} From 3d15da0f1569bb8366a98a5701ea0d58f8f2b468 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Tue, 7 Jun 2022 18:35:37 +0200 Subject: [PATCH 40/52] Cleanup, comments --- cli/src/api/mod.rs | 1 - cli/src/bin/phylum.rs | 13 ++---- cli/src/commands/extensions/api.rs | 46 +++++++++++-------- .../fixtures/extensions/api-extension/main.ts | 30 ++++++------ .../extensions/api-extension/phylum-api.ts | 5 ++ 5 files changed, 53 insertions(+), 42 deletions(-) diff --git a/cli/src/api/mod.rs b/cli/src/api/mod.rs index 6180fb67a..176f84107 100644 --- a/cli/src/api/mod.rs +++ b/cli/src/api/mod.rs @@ -33,7 +33,6 @@ type Result = std::result::Result; pub struct PhylumApi { config: Config, client: Client, - //api_uri: String, ignore_certs: bool, } diff --git a/cli/src/bin/phylum.rs b/cli/src/bin/phylum.rs index f8343491c..dd7de8f7f 100644 --- a/cli/src/bin/phylum.rs +++ b/cli/src/bin/phylum.rs @@ -55,14 +55,9 @@ async fn api_factory( timeout: Option, ignore_certs: bool, ) -> Result { - let api = PhylumApi::new( - // &mut config.auth_info, - config, - timeout, - ignore_certs, - ) - .await - .context("Error creating client")?; + let api = PhylumApi::new(config, timeout, ignore_certs) + .await + .context("Error creating client")?; // PhylumApi may have had to log in, updating the auth info so we should save the config save_config(&config_path, api.config()).with_context(|| { @@ -194,7 +189,7 @@ async fn handle_commands() -> CommandResult { "extension" => handle_extensions(matches).await, #[cfg(feature = "extensions")] - extension_subcmd => handle_run_extension(extension_subcmd, Box::pin(api)).await, // Extension::load(extension_subcmd)?.run().await, + extension_subcmd => handle_run_extension(extension_subcmd, Box::pin(api)).await, #[cfg(not(feature = "extensions"))] _ => unreachable!(), diff --git a/cli/src/commands/extensions/api.rs b/cli/src/commands/extensions/api.rs index 77ea4ea1d..60a4d448c 100644 --- a/cli/src/commands/extensions/api.rs +++ b/cli/src/commands/extensions/api.rs @@ -29,22 +29,22 @@ use phylum_types::types::package::{ }; use phylum_types::types::project::ProjectDetailsResponse; -// BROKEN - +/// Holds either an unawaited, boxed `Future`, or the result of awaiting the future. +/// In other words, it is a lazy evaluator for the `Future`. pub(crate) enum OnceFuture { - Future(Option>), + Future(BoxFuture<'static, T>), Awaited(T), } impl OnceFuture { fn new(t: BoxFuture<'static, T>) -> Self { - OnceFuture::Future(Some(t)) + OnceFuture::Future(t) } async fn get(&mut self) -> &T { match *self { OnceFuture::Future(ref mut t) => { - *self = OnceFuture::Awaited(t.take().unwrap().await); + *self = OnceFuture::Awaited(t.await); match *self { OnceFuture::Future(..) => unreachable!(), OnceFuture::Awaited(ref mut t) => t, @@ -55,9 +55,13 @@ impl OnceFuture { } } +/// Opaquely encapsulates the extension state. pub struct ExtensionState(OnceFuture>); impl ExtensionState { + /// Borrow an instance of `ExtensionState` from a reference to Deno's + /// `OpState` state handler, and retrieve a reference to the inner state. + /// The references lives as long as `&mut OpState`. async fn borrow_from(state: &mut OpState) -> Result<&PhylumApi> { state .borrow_mut::() @@ -75,15 +79,25 @@ impl From>> for ExtensionState { } } +// +// Extension API functions +// These functions need not be public, as Deno's declarations (`::decl()`) cloak +// them in a data structure that is consumed by the runtime extension builder. +// + /// Analyze a lockfile. /// Equivalent to `phylum analyze`. #[op] -pub(crate) async fn analyze( +async fn analyze( state: Rc>, lockfile: &str, project: Option<&str>, group: Option<&str>, ) -> Result { + // The general pattern for accessing state in extensions API functions is: + // - borrow from Deno's state handler + // - pin the borrowed reference guard + // - extract `Result<&PhylumApi>` from ExtensionState let mut state = Pin::new(state.borrow_mut()); let api = ExtensionState::borrow_from(&mut state).await?; @@ -118,7 +132,7 @@ pub(crate) async fn analyze( /// Retrieve user info. /// Equivalent to `phylum auth status`. #[op] -pub(crate) async fn get_user_info(state: Rc>) -> Result { +async fn get_user_info(state: Rc>) -> Result { let mut state = Pin::new(state.borrow_mut()); let api = ExtensionState::borrow_from(&mut state).await?; @@ -128,10 +142,7 @@ pub(crate) async fn get_user_info(state: Rc>) -> Result>, - ignore_certs: bool, -) -> Result { +async fn get_access_token(state: Rc>, ignore_certs: bool) -> Result { let refresh_token = get_refresh_token::call(state.clone()).await?; let mut state = Pin::new(state.borrow_mut()); let config = ExtensionState::borrow_from(&mut state).await?.config(); @@ -146,7 +157,7 @@ pub(crate) async fn get_access_token( /// Retrieve the refresh token. /// Equivalent to `phylum auth token`. #[op] -pub(crate) async fn get_refresh_token(state: Rc>) -> Result { +async fn get_refresh_token(state: Rc>) -> Result { let mut state = Pin::new(state.borrow_mut()); let config = ExtensionState::borrow_from(&mut state).await?.config(); @@ -160,7 +171,7 @@ pub(crate) async fn get_refresh_token(state: Rc>) -> Result>, job_id: Option<&str>, ) -> Result> { @@ -177,7 +188,7 @@ pub(crate) async fn get_job_status( /// Retrieve a project's details. /// Equivalent to `phylum history project`. #[op] -pub(crate) async fn get_project_details( +async fn get_project_details( state: Rc>, project_name: Option<&str>, ) -> Result { @@ -200,7 +211,7 @@ pub(crate) async fn get_project_details( /// Analyze a single package. /// Equivalent to `phylum package`. #[op] -pub(crate) async fn analyze_package( +async fn analyze_package( state: Rc>, name: &str, version: &str, @@ -223,10 +234,7 @@ pub(crate) async fn analyze_package( /// Parse a lockfile and return the package descriptors contained therein. /// Equivalent to `phylum parse`. #[op] -pub(crate) fn parse_lockfile( - lockfile: &str, - lockfile_type: &str, -) -> Result> { +fn parse_lockfile(lockfile: &str, lockfile_type: &str) -> Result> { let parser = LOCKFILE_PARSERS .iter() .find_map(|(name, parser)| (*name == lockfile_type).then(|| *parser)) diff --git a/cli/tests/fixtures/extensions/api-extension/main.ts b/cli/tests/fixtures/extensions/api-extension/main.ts index 2746832a1..8da66617c 100644 --- a/cli/tests/fixtures/extensions/api-extension/main.ts +++ b/cli/tests/fixtures/extensions/api-extension/main.ts @@ -1,15 +1,19 @@ import * as PhylumApi from './phylum-api.ts' -Deno.core.print(JSON.stringify(await PhylumApi.get_user_info())) -await PhylumApi.get_access_token() -await PhylumApi.get_refresh_token() -try { - await PhylumApi.get_job_status() -} catch(e) { - Deno.core.print(JSON.stringify(e)) -} -try { - await PhylumApi.get_project_details() -} catch(e) { - Deno.core.print(JSON.stringify(e)) -} +// Extension tests should be ran against an integration context, and not in CI +// at least for the moment, as they require access to an API instance, or a +// well-formed, external mock of it. + +// Deno.core.print(JSON.stringify(await PhylumApi.get_user_info())) +// await PhylumApi.get_access_token() +// await PhylumApi.get_refresh_token() +// try { +// await PhylumApi.get_job_status() +// } catch(e) { +// Deno.core.print(JSON.stringify(e)) +// } +// try { +// await PhylumApi.get_project_details() +// } catch(e) { +// Deno.core.print(JSON.stringify(e)) +// } diff --git a/cli/tests/fixtures/extensions/api-extension/phylum-api.ts b/cli/tests/fixtures/extensions/api-extension/phylum-api.ts index 3df11be4b..341df56ae 100644 --- a/cli/tests/fixtures/extensions/api-extension/phylum-api.ts +++ b/cli/tests/fixtures/extensions/api-extension/phylum-api.ts @@ -1,3 +1,8 @@ +// This helper file wraps Deno.core.opAsync calls for ergonomics. There is no +// concept of injecting a module in Deno, so clients won't be able to use something +// like `import * as PhylumApi from 'phylum-api'` and will have to manually include +// this file instead. + export async function analyze(lockfile: string, project?: string, group?: string) { return await Deno.core.opAsync("analyze", lockfile, project, group) } From 67e33ee6f00565b2510b2565fcc8efddceb3c278 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Wed, 8 Jun 2022 14:02:17 +0200 Subject: [PATCH 41/52] Cleanup deps insertion into Deno --- cli/src/deno.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/cli/src/deno.rs b/cli/src/deno.rs index 65a28b1e2..b6c675870 100644 --- a/cli/src/deno.rs +++ b/cli/src/deno.rs @@ -29,9 +29,7 @@ impl DenoRuntime { ..Default::default() }); - let op_state = runtime.op_state(); - let mut op_state = op_state.borrow_mut(); - op_state.put(deps); + runtime.op_state().borrow_mut().put(deps); Self { runtime } } From 5d6fea06024d28cdc96184f616ebf7dc21494d64 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Wed, 8 Jun 2022 18:45:57 +0200 Subject: [PATCH 42/52] Removed "allow unused" and cleaned up --- cli/src/commands/extensions/api.rs | 14 ++------------ cli/src/commands/extensions/mod.rs | 4 ++++ 2 files changed, 6 insertions(+), 12 deletions(-) diff --git a/cli/src/commands/extensions/api.rs b/cli/src/commands/extensions/api.rs index 60a4d448c..4f48753be 100644 --- a/cli/src/commands/extensions/api.rs +++ b/cli/src/commands/extensions/api.rs @@ -1,26 +1,16 @@ -//! TODO remove the following annotation before merging. The functions in -//! this module should have as unique client the Deno runtime; pending its -//! implementation, having these functions unused would break CI. -#![allow(unused)] - -use std::cell::{RefCell, RefMut}; -use std::future::Future; +use std::cell::RefCell; use std::path::Path; use std::pin::Pin; use std::rc::Rc; use std::str::FromStr; -use std::sync::Arc; use crate::commands::parse::{get_packages_from_lockfile, LOCKFILE_PARSERS}; -use crate::config::{get_current_project, Config}; -use crate::lockfiles::Parse; +use crate::config::get_current_project; use crate::{api::PhylumApi, auth::UserInfo}; use anyhow::{anyhow, Context, Error, Result}; -use deno_core::parking_lot::{MappedMutexGuard, Mutex, MutexGuard}; use deno_core::{op, OpDecl, OpState}; use futures::future::BoxFuture; -use once_cell::sync::Lazy; use phylum_types::types::auth::{AccessToken, RefreshToken}; use phylum_types::types::common::{JobId, ProjectId}; use phylum_types::types::job::JobStatusResponse; diff --git a/cli/src/commands/extensions/mod.rs b/cli/src/commands/extensions/mod.rs index 53f824e1d..8f37b3399 100644 --- a/cli/src/commands/extensions/mod.rs +++ b/cli/src/commands/extensions/mod.rs @@ -77,6 +77,7 @@ pub async fn handle_extensions(matches: &ArgMatches) -> CommandResult { } /// Handle the `` command path. +/// /// Run the extension by name. pub async fn handle_run_extension( name: &str, @@ -90,6 +91,7 @@ pub async fn handle_run_extension( } /// Handle the `extension add` subcommand path. +/// /// Add the extension from the specified path. async fn handle_add_extension(path: &str) -> CommandResult { // NOTE: Extension installation without slashes is reserved for the marketplace. @@ -109,6 +111,7 @@ async fn handle_add_extension(path: &str) -> CommandResult { } /// Handle the `extension remove` subcommand path. +/// /// Remove the extension named as specified. async fn handle_remove_extension(name: &str) -> CommandResult { let extension = Extension::load(name)?; @@ -119,6 +122,7 @@ async fn handle_remove_extension(name: &str) -> CommandResult { } /// Handle the `extension` / `extension list` subcommand path. +/// /// List installed extensions. async fn handle_list_extensions() -> CommandResult { let extensions = installed_extensions()?; From 59b1160b9660e5b212994361f8dc2c3f976f23c5 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Wed, 8 Jun 2022 18:59:16 +0200 Subject: [PATCH 43/52] Removed test --- cli/tests/extensions.rs | 30 ------------------------------ 1 file changed, 30 deletions(-) diff --git a/cli/tests/extensions.rs b/cli/tests/extensions.rs index 67a00d5c0..a009e8d82 100644 --- a/cli/tests/extensions.rs +++ b/cli/tests/extensions.rs @@ -261,36 +261,6 @@ fn conflicting_extension_name_is_filtered() { assert!(output.contains("extension was filtered out")); } -#[test] -fn api_extension() { - let tmp_dir = TmpDir::new(); - Command::cargo_bin("phylum") - .unwrap() - .env("XDG_DATA_HOME", &tmp_dir) - .arg("extension") - .arg("add") - .arg(fixtures_path().join("api-extension")) - .assert() - .success(); - - let cmd = Command::cargo_bin("phylum") - .unwrap() - .env("XDG_DATA_HOME", &tmp_dir) - .arg("api-extension") - .assert() - .success(); - - { - let output = std::str::from_utf8(&cmd.get_output().stdout).unwrap(); - println!("{}", output); - } - { - let output = std::str::from_utf8(&cmd.get_output().stderr).unwrap(); - println!("{}", output); - } - // assert!(output.contains("extension was filtered out")); -} - //////////////////////////////////////////////////////////////////////////////// // Utilities //////////////////////////////////////////////////////////////////////////////// From fcb1eda854ab809d6519e54d0a572520400facd3 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Wed, 8 Jun 2022 19:04:16 +0200 Subject: [PATCH 44/52] Refactored var name --- cli/src/commands/extensions/api.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cli/src/commands/extensions/api.rs b/cli/src/commands/extensions/api.rs index 4f48753be..289d3dd23 100644 --- a/cli/src/commands/extensions/api.rs +++ b/cli/src/commands/extensions/api.rs @@ -27,20 +27,20 @@ pub(crate) enum OnceFuture { } impl OnceFuture { - fn new(t: BoxFuture<'static, T>) -> Self { - OnceFuture::Future(t) + fn new(inner: BoxFuture<'static, T>) -> Self { + OnceFuture::Future(inner) } async fn get(&mut self) -> &T { match *self { - OnceFuture::Future(ref mut t) => { - *self = OnceFuture::Awaited(t.await); + OnceFuture::Future(ref mut inner) => { + *self = OnceFuture::Awaited(inner.await); match *self { OnceFuture::Future(..) => unreachable!(), - OnceFuture::Awaited(ref mut t) => t, + OnceFuture::Awaited(ref mut inner) => inner, } } - OnceFuture::Awaited(ref mut t) => t, + OnceFuture::Awaited(ref mut inner) => inner, } } } From 837a455ba7cff6be5c623125304ba15cd39e3720 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Wed, 8 Jun 2022 19:04:33 +0200 Subject: [PATCH 45/52] Cleaned comment --- cli/src/commands/extensions/api.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/cli/src/commands/extensions/api.rs b/cli/src/commands/extensions/api.rs index 289d3dd23..8b3ff36b8 100644 --- a/cli/src/commands/extensions/api.rs +++ b/cli/src/commands/extensions/api.rs @@ -20,7 +20,6 @@ use phylum_types::types::package::{ use phylum_types::types::project::ProjectDetailsResponse; /// Holds either an unawaited, boxed `Future`, or the result of awaiting the future. -/// In other words, it is a lazy evaluator for the `Future`. pub(crate) enum OnceFuture { Future(BoxFuture<'static, T>), Awaited(T), From d60a1efee42f529d409df37450cdcf78e0e8700d Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Wed, 8 Jun 2022 19:31:21 +0200 Subject: [PATCH 46/52] Moved and expanded comment --- cli/src/commands/extensions/api.rs | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/cli/src/commands/extensions/api.rs b/cli/src/commands/extensions/api.rs index 8b3ff36b8..3360366f0 100644 --- a/cli/src/commands/extensions/api.rs +++ b/cli/src/commands/extensions/api.rs @@ -73,6 +73,17 @@ impl From>> for ExtensionState { // These functions need not be public, as Deno's declarations (`::decl()`) cloak // them in a data structure that is consumed by the runtime extension builder. // +// The general pattern for accessing state in extensions API functions is: +// - borrow from Deno's state handler +// - pin the borrowed reference guard +// - extract `Result<&PhylumApi>` from ExtensionState +// +// The borrow is necessary as Deno's state is held inside of a `RefCell`. The resulting mutable +// reference is going to be held across an await point, which means it should be pinned to prevent +// it being moved, because that would result in an invalid reference. The extracted +// `Result<&PhylumApi>` reference lives as long as the pinned `RefMut` and can be used until both +// go out of scope and the guard is released. +// /// Analyze a lockfile. /// Equivalent to `phylum analyze`. @@ -83,10 +94,6 @@ async fn analyze( project: Option<&str>, group: Option<&str>, ) -> Result { - // The general pattern for accessing state in extensions API functions is: - // - borrow from Deno's state handler - // - pin the borrowed reference guard - // - extract `Result<&PhylumApi>` from ExtensionState let mut state = Pin::new(state.borrow_mut()); let api = ExtensionState::borrow_from(&mut state).await?; From 5ccea39163c675e4a2fc317ddfb6aa6f80f987a6 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Fri, 10 Jun 2022 15:49:43 +0200 Subject: [PATCH 47/52] ExtensionStateRef wrapper type --- cli/src/commands/extensions/api.rs | 83 +++++++++++++++--------------- 1 file changed, 42 insertions(+), 41 deletions(-) diff --git a/cli/src/commands/extensions/api.rs b/cli/src/commands/extensions/api.rs index 3360366f0..bced98b2f 100644 --- a/cli/src/commands/extensions/api.rs +++ b/cli/src/commands/extensions/api.rs @@ -1,16 +1,14 @@ use std::cell::RefCell; +use std::ops::Deref; use std::path::Path; use std::pin::Pin; use std::rc::Rc; use std::str::FromStr; -use crate::commands::parse::{get_packages_from_lockfile, LOCKFILE_PARSERS}; -use crate::config::get_current_project; -use crate::{api::PhylumApi, auth::UserInfo}; - use anyhow::{anyhow, Context, Error, Result}; use deno_core::{op, OpDecl, OpState}; use futures::future::BoxFuture; + use phylum_types::types::auth::{AccessToken, RefreshToken}; use phylum_types::types::common::{JobId, ProjectId}; use phylum_types::types::job::JobStatusResponse; @@ -19,6 +17,10 @@ use phylum_types::types::package::{ }; use phylum_types::types::project::ProjectDetailsResponse; +use crate::commands::parse::{get_packages_from_lockfile, LOCKFILE_PARSERS}; +use crate::config::get_current_project; +use crate::{api::PhylumApi, auth::UserInfo}; + /// Holds either an unawaited, boxed `Future`, or the result of awaiting the future. pub(crate) enum OnceFuture { Future(BoxFuture<'static, T>), @@ -45,26 +47,41 @@ impl OnceFuture { } /// Opaquely encapsulates the extension state. -pub struct ExtensionState(OnceFuture>); - -impl ExtensionState { - /// Borrow an instance of `ExtensionState` from a reference to Deno's - /// `OpState` state handler, and retrieve a reference to the inner state. - /// The references lives as long as `&mut OpState`. - async fn borrow_from(state: &mut OpState) -> Result<&PhylumApi> { - state +pub struct ExtensionState(OnceFuture>>); + +impl From>> for ExtensionState { + fn from(t: BoxFuture<'static, Result>) -> Self { + Self(OnceFuture::new(Box::pin(async { t.await.map(Rc::new) }))) + } +} + +/// Wraps a shared, counted reference to the `PhylumApi` object. The reference can be safely +/// extracted from `Rc>` and cloned; it will not require mutable access to the +/// owning `RefCell`, so the mutable borrow to it may be dropped. +struct ExtensionStateRef(Rc); + +impl ExtensionStateRef { + // This can not be implemented as the `From` trait because of `async`. + async fn from(state: Rc>) -> Result { + let mut state_ref = Pin::new(state.try_borrow_mut()?); + let api = state_ref .borrow_mut::() .0 .get() .await .as_ref() - .map_err(|e| anyhow!("{:?}", e)) + .map_err(|e| anyhow!("{:?}", e))? + .clone(); + + Ok(ExtensionStateRef(api)) } } -impl From>> for ExtensionState { - fn from(t: BoxFuture<'static, Result>) -> Self { - Self(OnceFuture::new(t)) +impl Deref for ExtensionStateRef { + type Target = PhylumApi; + + fn deref(&self) -> &Self::Target { + &self.0 } } @@ -73,17 +90,6 @@ impl From>> for ExtensionState { // These functions need not be public, as Deno's declarations (`::decl()`) cloak // them in a data structure that is consumed by the runtime extension builder. // -// The general pattern for accessing state in extensions API functions is: -// - borrow from Deno's state handler -// - pin the borrowed reference guard -// - extract `Result<&PhylumApi>` from ExtensionState -// -// The borrow is necessary as Deno's state is held inside of a `RefCell`. The resulting mutable -// reference is going to be held across an await point, which means it should be pinned to prevent -// it being moved, because that would result in an invalid reference. The extracted -// `Result<&PhylumApi>` reference lives as long as the pinned `RefMut` and can be used until both -// go out of scope and the guard is released. -// /// Analyze a lockfile. /// Equivalent to `phylum analyze`. @@ -94,8 +100,7 @@ async fn analyze( project: Option<&str>, group: Option<&str>, ) -> Result { - let mut state = Pin::new(state.borrow_mut()); - let api = ExtensionState::borrow_from(&mut state).await?; + let api = ExtensionStateRef::from(state).await?; let (packages, request_type) = get_packages_from_lockfile(Path::new(lockfile)) .context("Unable to locate any valid package in package lockfile")?; @@ -129,8 +134,7 @@ async fn analyze( /// Equivalent to `phylum auth status`. #[op] async fn get_user_info(state: Rc>) -> Result { - let mut state = Pin::new(state.borrow_mut()); - let api = ExtensionState::borrow_from(&mut state).await?; + let api = ExtensionStateRef::from(state).await?; api.user_info().await.map_err(Error::from) } @@ -140,8 +144,8 @@ async fn get_user_info(state: Rc>) -> Result { #[op] async fn get_access_token(state: Rc>, ignore_certs: bool) -> Result { let refresh_token = get_refresh_token::call(state.clone()).await?; - let mut state = Pin::new(state.borrow_mut()); - let config = ExtensionState::borrow_from(&mut state).await?.config(); + let api = ExtensionStateRef::from(state).await?; + let config = api.config(); let access_token = crate::auth::handle_refresh_tokens(&refresh_token, ignore_certs, &config.connection.uri) @@ -154,8 +158,8 @@ async fn get_access_token(state: Rc>, ignore_certs: bool) -> Re /// Equivalent to `phylum auth token`. #[op] async fn get_refresh_token(state: Rc>) -> Result { - let mut state = Pin::new(state.borrow_mut()); - let config = ExtensionState::borrow_from(&mut state).await?.config(); + let api = ExtensionStateRef::from(state).await?; + let config = api.config(); config .auth_info @@ -171,8 +175,7 @@ async fn get_job_status( state: Rc>, job_id: Option<&str>, ) -> Result> { - let mut state = Pin::new(state.borrow_mut()); - let api = ExtensionState::borrow_from(&mut state).await?; + let api = ExtensionStateRef::from(state).await?; let job_id = job_id .map(|job_id| JobId::from_str(job_id).ok()) @@ -188,8 +191,7 @@ async fn get_project_details( state: Rc>, project_name: Option<&str>, ) -> Result { - let mut state = Pin::new(state.borrow_mut()); - let api = ExtensionState::borrow_from(&mut state).await?; + let api = ExtensionStateRef::from(state).await?; let project_name = project_name .map(String::from) @@ -213,8 +215,7 @@ async fn analyze_package( version: &str, package_type: &str, ) -> Result { - let mut state = Pin::new(state.borrow_mut()); - let api = ExtensionState::borrow_from(&mut state).await?; + let api = ExtensionStateRef::from(state).await?; let package_type = PackageType::from_str(package_type) .map_err(|e| anyhow!("Unrecognized package type `{package_type}`: {e:?}"))?; From 699b1dcabaed19dd253fb56adca44fddc5382a41 Mon Sep 17 00:00:00 2001 From: andreaphylum <68552656+andreaphylum@users.noreply.github.com> Date: Mon, 13 Jun 2022 15:18:50 +0200 Subject: [PATCH 48/52] Update cli/src/commands/extensions/api.rs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Christian Dürr <102963075+cd-work@users.noreply.github.com> --- cli/src/commands/extensions/api.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/src/commands/extensions/api.rs b/cli/src/commands/extensions/api.rs index bced98b2f..49d231cd7 100644 --- a/cli/src/commands/extensions/api.rs +++ b/cli/src/commands/extensions/api.rs @@ -55,9 +55,9 @@ impl From>> for ExtensionState { } } -/// Wraps a shared, counted reference to the `PhylumApi` object. The reference can be safely -/// extracted from `Rc>` and cloned; it will not require mutable access to the -/// owning `RefCell`, so the mutable borrow to it may be dropped. +/// Wraps a shared, counted reference to the `PhylumApi` object. +/// +/// The reference can be safely extracted from `Rc>` and cloned; it will not require mutable access to the owning `RefCell`, so the mutable borrow to it may be dropped. struct ExtensionStateRef(Rc); impl ExtensionStateRef { From 2af429b5ab5c6f5f6d6c3b31109a01babfa675a4 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Mon, 13 Jun 2022 15:20:12 +0200 Subject: [PATCH 49/52] Applied change requests --- cli/src/commands/extensions/api.rs | 8 +++++--- cli/src/commands/extensions/mod.rs | 7 +++---- cli/src/deno.rs | 6 +++--- cli/tests/extensions.rs | 2 +- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/cli/src/commands/extensions/api.rs b/cli/src/commands/extensions/api.rs index bced98b2f..ac7fb456d 100644 --- a/cli/src/commands/extensions/api.rs +++ b/cli/src/commands/extensions/api.rs @@ -22,7 +22,7 @@ use crate::config::get_current_project; use crate::{api::PhylumApi, auth::UserInfo}; /// Holds either an unawaited, boxed `Future`, or the result of awaiting the future. -pub(crate) enum OnceFuture { +pub enum OnceFuture { Future(BoxFuture<'static, T>), Awaited(T), } @@ -50,8 +50,10 @@ impl OnceFuture { pub struct ExtensionState(OnceFuture>>); impl From>> for ExtensionState { - fn from(t: BoxFuture<'static, Result>) -> Self { - Self(OnceFuture::new(Box::pin(async { t.await.map(Rc::new) }))) + fn from(extension_state_future: BoxFuture<'static, Result>) -> Self { + Self(OnceFuture::new(Box::pin(async { + extension_state_future.await.map(Rc::new) + }))) } } diff --git a/cli/src/commands/extensions/mod.rs b/cli/src/commands/extensions/mod.rs index 8f37b3399..52bc3ef25 100644 --- a/cli/src/commands/extensions/mod.rs +++ b/cli/src/commands/extensions/mod.rs @@ -1,8 +1,7 @@ -mod api; -mod extension; +pub mod api; +pub mod extension; -pub(crate) use api::api_decls; -pub use extension::*; +use extension::*; use std::{collections::HashSet, fs, io::ErrorKind, path::PathBuf}; diff --git a/cli/src/deno.rs b/cli/src/deno.rs index b6c675870..5039e7b4f 100644 --- a/cli/src/deno.rs +++ b/cli/src/deno.rs @@ -11,7 +11,7 @@ use deno_core::{ ModuleType, RuntimeOptions, }; -use crate::commands::extensions::{api_decls, ExtensionState}; +use crate::commands::extensions::{api::api_decls, extension::ExtensionState}; /// Deno runtime state. pub struct DenoRuntime { @@ -20,7 +20,7 @@ pub struct DenoRuntime { impl DenoRuntime { /// Create a new Deno runtime. - pub fn new(deps: ExtensionState) -> Self { + pub fn new(extension_state: ExtensionState) -> Self { let phylum_api = Extension::builder().ops(api_decls()).build(); let mut runtime = JsRuntime::new(RuntimeOptions { @@ -29,7 +29,7 @@ impl DenoRuntime { ..Default::default() }); - runtime.op_state().borrow_mut().put(deps); + runtime.op_state().borrow_mut().put(extension_state); Self { runtime } } diff --git a/cli/tests/extensions.rs b/cli/tests/extensions.rs index a009e8d82..9a04b8644 100644 --- a/cli/tests/extensions.rs +++ b/cli/tests/extensions.rs @@ -9,7 +9,7 @@ use std::sync::Mutex; use assert_cmd::Command; use lazy_static::lazy_static; -use phylum_cli::commands::extensions::*; +use phylum_cli::commands::extensions::extension::Extension; use rand::Rng; use regex::Regex; From c2d12d3d06bd7df665fd10fdea503641aa6529b6 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Mon, 13 Jun 2022 18:11:40 +0200 Subject: [PATCH 50/52] Removed pub qualifier --- cli/src/commands/extensions/api.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/src/commands/extensions/api.rs b/cli/src/commands/extensions/api.rs index 8691fab1e..c93ea9010 100644 --- a/cli/src/commands/extensions/api.rs +++ b/cli/src/commands/extensions/api.rs @@ -22,7 +22,7 @@ use crate::config::get_current_project; use crate::{api::PhylumApi, auth::UserInfo}; /// Holds either an unawaited, boxed `Future`, or the result of awaiting the future. -pub enum OnceFuture { +enum OnceFuture { Future(BoxFuture<'static, T>), Awaited(T), } From ef38455f620c666d028a5ac27ed05885e5e87814 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Wed, 15 Jun 2022 19:15:15 +0200 Subject: [PATCH 51/52] Reverted trait generic removal --- cli/src/commands/extensions/api.rs | 3 ++- cli/src/lockfiles/csharp.rs | 8 ++------ cli/src/lockfiles/java.rs | 8 ++------ cli/src/lockfiles/javascript.rs | 16 ++++++---------- cli/src/lockfiles/mod.rs | 5 ++++- cli/src/lockfiles/python.rs | 22 ++++++---------------- cli/src/lockfiles/ruby.rs | 6 +----- 7 files changed, 23 insertions(+), 45 deletions(-) diff --git a/cli/src/commands/extensions/api.rs b/cli/src/commands/extensions/api.rs index c93ea9010..f3cfa9be1 100644 --- a/cli/src/commands/extensions/api.rs +++ b/cli/src/commands/extensions/api.rs @@ -239,7 +239,8 @@ fn parse_lockfile(lockfile: &str, lockfile_type: &str) -> Result Vec { diff --git a/cli/src/lockfiles/csharp.rs b/cli/src/lockfiles/csharp.rs index 5b59db179..9610c4558 100644 --- a/cli/src/lockfiles/csharp.rs +++ b/cli/src/lockfiles/csharp.rs @@ -76,15 +76,11 @@ impl Parse for CSProj { #[cfg(test)] mod tests { - use std::path::Path; - use super::*; #[test] fn lock_parse_csproj() { - let pkgs = CSProj - .parse_file(Path::new("tests/fixtures/sample.csproj")) - .unwrap(); + let pkgs = CSProj.parse_file("tests/fixtures/sample.csproj").unwrap(); assert_eq!(pkgs.len(), 5); assert_eq!(pkgs[0].name, "Microsoft.NETFramework.ReferenceAssemblies"); @@ -100,7 +96,7 @@ mod tests { #[test] fn lock_parse_another_invalid_char() { let pkgs = CSProj - .parse_file(Path::new("tests/fixtures/Calculator.csproj")) + .parse_file("tests/fixtures/Calculator.csproj") .unwrap(); assert!(!pkgs.is_empty()); } diff --git a/cli/src/lockfiles/java.rs b/cli/src/lockfiles/java.rs index b842fd438..a9af772ab 100644 --- a/cli/src/lockfiles/java.rs +++ b/cli/src/lockfiles/java.rs @@ -136,14 +136,12 @@ impl Parse for Pom { #[cfg(test)] mod tests { - use std::path::Path; - use super::*; #[test] fn lock_parse_gradle() { let pkgs = GradleLock - .parse_file(Path::new("tests/fixtures/gradle.lockfile")) + .parse_file("tests/fixtures/gradle.lockfile") .unwrap(); assert_eq!(pkgs.len(), 5); @@ -159,9 +157,7 @@ mod tests { #[test] fn lock_parse_effective_pom() { - let mut pkgs = Pom - .parse_file(Path::new("tests/fixtures/effective-pom.xml")) - .unwrap(); + let mut pkgs = Pom.parse_file("tests/fixtures/effective-pom.xml").unwrap(); pkgs.sort_by(|a, b| a.version.cmp(&b.version)); assert_eq!(pkgs.len(), 16); diff --git a/cli/src/lockfiles/javascript.rs b/cli/src/lockfiles/javascript.rs index 970aeaf46..c89aa75bc 100644 --- a/cli/src/lockfiles/javascript.rs +++ b/cli/src/lockfiles/javascript.rs @@ -167,14 +167,12 @@ impl Parse for YarnLock { #[cfg(test)] mod tests { - use std::path::Path; - use super::*; #[test] fn lock_parse_package() { let pkgs = PackageLock - .parse_file(Path::new("tests/fixtures/package-lock-v6.json")) + .parse_file("tests/fixtures/package-lock-v6.json") .unwrap(); assert_eq!(pkgs.len(), 17); @@ -191,7 +189,7 @@ mod tests { #[test] fn lock_parse_package_v7() { let pkgs = PackageLock - .parse_file(Path::new("tests/fixtures/package-lock.json")) + .parse_file("tests/fixtures/package-lock.json") .unwrap(); assert_eq!(pkgs.len(), 50); @@ -221,7 +219,7 @@ mod tests { // We need to make sure we don't take the v2 lockfile code path because this is not a v2 // lockfile and parsing it as one will produce incorrect results. let pkgs = YarnLock - .parse_file(Path::new("tests/fixtures/yarn-v1.simple.lock")) + .parse_file("tests/fixtures/yarn-v1.simple.lock") .unwrap(); assert_eq!( @@ -240,7 +238,7 @@ mod tests { "tests/fixtures/yarn-v1.lock", "tests/fixtures/yarn-v1.trailing_newlines.lock", ] { - let pkgs = YarnLock.parse_file(Path::new(p)).unwrap(); + let pkgs = YarnLock.parse_file(p).unwrap(); assert_eq!(pkgs.len(), 17); @@ -263,15 +261,13 @@ mod tests { #[test] fn lock_parse_yarn_v1_malformed_fails() { YarnLock - .parse_file(Path::new("tests/fixtures/yarn-v1.lock.bad")) + .parse_file("tests/fixtures/yarn-v1.lock.bad") .unwrap(); } #[test] fn lock_parse_yarn() { - let pkgs = YarnLock - .parse_file(Path::new("tests/fixtures/yarn.lock")) - .unwrap(); + let pkgs = YarnLock.parse_file("tests/fixtures/yarn.lock").unwrap(); assert_eq!(pkgs.len(), 53); diff --git a/cli/src/lockfiles/mod.rs b/cli/src/lockfiles/mod.rs index 944fb5672..faea6f7da 100644 --- a/cli/src/lockfiles/mod.rs +++ b/cli/src/lockfiles/mod.rs @@ -24,7 +24,10 @@ pub trait Parse { fn parse(&self, data: &str) -> ParseResult; /// Parse from a file - fn parse_file(&self, path: &Path) -> ParseResult { + fn parse_file>(&self, path: P) -> ParseResult + where + Self: Sized, + { let data = read_to_string(path)?; self.parse(&data) } diff --git a/cli/src/lockfiles/python.rs b/cli/src/lockfiles/python.rs index ac40ea113..d5b1b5af3 100644 --- a/cli/src/lockfiles/python.rs +++ b/cli/src/lockfiles/python.rs @@ -167,14 +167,12 @@ struct PoetryMetadata { #[cfg(test)] mod tests { - use std::path::Path; - use super::*; #[test] fn parse_requirements() { let pkgs = PyRequirements - .parse_file(Path::new("tests/fixtures/requirements.txt")) + .parse_file("tests/fixtures/requirements.txt") .unwrap(); assert_eq!(pkgs.len(), 131); assert_eq!(pkgs[0].name, "pyyaml"); @@ -190,7 +188,7 @@ mod tests { #[test] fn parse_requirements_complex() { let pkgs = PyRequirements - .parse_file(Path::new("tests/fixtures/complex-requirements.txt")) + .parse_file("tests/fixtures/complex-requirements.txt") .unwrap(); assert_eq!(pkgs.len(), 8); assert_eq!(pkgs[0].name, "docopt"); @@ -208,9 +206,7 @@ mod tests { #[test] fn parse_pipfile() { - let pkgs = PipFile - .parse_file(Path::new("tests/fixtures/Pipfile")) - .unwrap(); + let pkgs = PipFile.parse_file("tests/fixtures/Pipfile").unwrap(); assert_eq!(pkgs.len(), 4); let expected_pkgs = [ @@ -238,9 +234,7 @@ mod tests { #[test] fn lock_parse_pipfile() { - let pkgs = PipFile - .parse_file(Path::new("tests/fixtures/Pipfile.lock")) - .unwrap(); + let pkgs = PipFile.parse_file("tests/fixtures/Pipfile.lock").unwrap(); assert_eq!(pkgs.len(), 27); let expected_pkgs = [ @@ -268,9 +262,7 @@ mod tests { #[test] fn parse_poetry_lock() { - let pkgs = Poetry - .parse_file(Path::new("tests/fixtures/poetry.lock")) - .unwrap(); + let pkgs = Poetry.parse_file("tests/fixtures/poetry.lock").unwrap(); assert_eq!(pkgs.len(), 44); let expected_pkgs = [ @@ -299,9 +291,7 @@ mod tests { /// Ensure sources other than PyPi are ignored. #[test] fn poetry_ignore_other_sources() { - let pkgs = Poetry - .parse_file(Path::new("tests/fixtures/poetry.lock")) - .unwrap(); + let pkgs = Poetry.parse_file("tests/fixtures/poetry.lock").unwrap(); let invalid_package_names = ["toml", "directory-test", "requests"]; for pkg in pkgs { diff --git a/cli/src/lockfiles/ruby.rs b/cli/src/lockfiles/ruby.rs index e3d1df641..04620a67d 100644 --- a/cli/src/lockfiles/ruby.rs +++ b/cli/src/lockfiles/ruby.rs @@ -25,17 +25,13 @@ impl Parse for GemLock { #[cfg(test)] mod tests { - use std::path::Path; - use phylum_types::types::package::PackageType; use super::*; #[test] fn lock_parse_gem() { - let pkgs = GemLock - .parse_file(Path::new("tests/fixtures/Gemfile.lock")) - .unwrap(); + let pkgs = GemLock.parse_file("tests/fixtures/Gemfile.lock").unwrap(); assert_eq!(pkgs.len(), 214); assert_eq!(pkgs[0].name, "CFPropertyList"); assert_eq!(pkgs[0].version, "2.3.6"); From 1cfc30cc4120bb3dc0d1804aeff24cb528197427 Mon Sep 17 00:00:00 2001 From: Andrea Venuta Date: Wed, 15 Jun 2022 19:20:45 +0200 Subject: [PATCH 52/52] Renamed analyze_package into get_package_details --- cli/src/commands/extensions/api.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cli/src/commands/extensions/api.rs b/cli/src/commands/extensions/api.rs index f3cfa9be1..a1ac63b38 100644 --- a/cli/src/commands/extensions/api.rs +++ b/cli/src/commands/extensions/api.rs @@ -211,7 +211,7 @@ async fn get_project_details( /// Analyze a single package. /// Equivalent to `phylum package`. #[op] -async fn analyze_package( +async fn get_package_details( state: Rc>, name: &str, version: &str, @@ -251,7 +251,7 @@ pub(crate) fn api_decls() -> Vec { get_refresh_token::decl(), get_job_status::decl(), get_project_details::decl(), - analyze_package::decl(), + get_package_details::decl(), parse_lockfile::decl(), ] }