diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..177567f --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,60 @@ +name: GitHub CI/CD Workflow + +on: [push, pull_request] + +jobs: + build: + + runs-on: ubuntu-latest + container: + image: ubuntu:24.04 + env: + OPENSSL_DIR: /usr/include/openssl + OPENSSL_LIB_DIR: /usr/lib/x86_64-linux-gnu + OPENSSL_INCLUDE_DIR: /usr/include/openssl + + steps: + - name: Install dependencies + run: | + apt-get update + apt-get install -y curl git build-essential libssl-dev + + - uses: actions/checkout@v4 # checkout the repository + - name: Install Rust + run: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + echo "$HOME/.cargo/bin" >> $GITHUB_PATH + + - name: Set up JDK 11 # required for build-models.sh + uses: actions/setup-java@v3 + with: + distribution: 'adopt' + java-version: '11' + + - name: Install Go + uses: actions/setup-go@v4 + with: + go-version: '^1.22' + - name: Install Funnel + run: | + if [ -d "funnel" ]; then rm -Rf funnel; fi + git clone https://github.com/ohsu-comp-bio/funnel.git + cd funnel && make && make install && cd .. + - uses: actions/checkout@v4 # checkout the repository + - name: Build models + run: | + bash ./build-models.sh + - name: Build + run: | + cargo build --verbose + - name: Run tests + run: | + bash ./run-tests.sh + - name: Lint + run: | + cargo clippy -- -D warnings + - name: Format + run: | + cargo fmt -- ./lib/src/serviceinfo/models/*.rs # workaround to fix autogenerated code formatting + cargo fmt -- ./lib/src/tes/models/*.rs + cargo fmt -- --check \ No newline at end of file diff --git a/.github/workflows/local.yml b/.github/workflows/local.yml new file mode 100644 index 0000000..d0b039f --- /dev/null +++ b/.github/workflows/local.yml @@ -0,0 +1,98 @@ +name: Local CI/CD Workflow + +on: workflow_dispatch + +jobs: + build: + + runs-on: ubuntu-latest + container: + image: ubuntu:24.04 + env: + OPENSSL_DIR: /usr/include/openssl + OPENSSL_LIB_DIR: /usr/lib/x86_64-linux-gnu + OPENSSL_INCLUDE_DIR: /usr/include/openssl + + steps: + + - name: Cache Rust dependencies + uses: actions/cache@v3 + with: + path: ~/.cargo + key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.lock') }} + restore-keys: ${{ runner.os }}-cargo- + + - name: Cache Rust build output + uses: actions/cache@v3 + with: + path: target + key: ${{ runner.os }}-build-${{ hashFiles('**/Cargo.lock') }} + restore-keys: ${{ runner.os }}-build- + + - name: Cache Maven dependencies + uses: actions/cache@v3 + with: + path: ~/.m2/repository + key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-maven- + + - name: Cache Go modules + uses: actions/cache@v3 + with: + path: ~/.cache/go-build + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: ${{ runner.os }}-go- + + - name: Cache Funnel dependencies + uses: actions/cache@v3 + with: + path: ~/funnel/build + key: ${{ runner.os }}-funnel-${{ hashFiles('**/funnel/*') }} + restore-keys: ${{ runner.os }}-funnel- + + - uses: actions/checkout@v4 # checkout the repository + - name: Install Rust + run: | + curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y + echo "$HOME/.cargo/bin" >> $GITHUB_PATH + + - name: Set up JDK 11 # required for build-models.sh + uses: actions/setup-java@v3 + with: + distribution: 'adopt' + java-version: '11' + + - name: Install Go + uses: actions/setup-go@v4 + with: + go-version: '^1.22' + - name: Install Funnel + run: | + if [ -d "funnel" ]; then rm -Rf funnel; fi + git clone https://github.com/ohsu-comp-bio/funnel.git + cd funnel && make && make install && cd .. + - uses: actions/checkout@v4 # checkout the repository + - name: Build models + run: | + . $HOME/.cargo/env + bash ./build-models.sh + - name: Build + run: | + . $HOME/.cargo/env + cargo build --verbose + - name: Run tests + run: | + . $HOME/.cargo/env + bash ./run-tests.sh + - name: Lint + run: | + . $HOME/.cargo/env + cargo clippy -- -D warnings + - name: Format + run: | + . $HOME/.cargo/env + # rustup install nightly – fails for some reason + # rustup default nightly + cargo fmt -- ./lib/src/serviceinfo/models/*.rs # workaround to fix autogenerated code formatting + cargo fmt -- ./lib/src/tes/models/*.rs + cargo fmt -- --check # --config-path ./rustfmt.toml \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..43b246d --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +target/ +debug/ +Cargo.lock +openapitools.json +*.log +funnel-work-dir/ +funnel/ +lib/src/*/models diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..e366e1a --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,13 @@ +[workspace] +members = [ + "cli", + "lib" +] + +[workspace.package] +version = "0.1.0" +authors = ["Aarav Mehta (Google Summer of Code Participant)", "Pavel Nikonorov (Google Summer of Code Mentor; GENXT)", "ELIXIR Cloud & AAI (ELIXIR Europe)"] +edition = "2021" +readme = "./README.md" +license-file = "LICENSE.txt" +repository = "https://github.com/elixir-cloud-aai/ga4gh-sdk.git" \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..6db902b --- /dev/null +++ b/README.md @@ -0,0 +1,36 @@ +# A Generic SDK and CLI for GA4GH API services + +## Building + +First, clone the repository, and then run the following command to automatically generate models using OpenAPI specifications: +``` +bash ./build-models.sh +``` + +To build the project: +``` +cargo build +``` + +## Running the tests + +Before running the tests, you need to install Funnel, a task execution system that is compatible with the GA4GH TES API. Follow the instructions in the [Funnel Developer's Guide](https://ohsu-comp-bio.github.io/funnel/docs/development/developers/) to install Funnel. + +Once you have installed Funnel, you can run the tests. This will automatically run Funnel as well: + +``` +bash ./run-tests.sh +``` +or, you can run using cargo nextest using +``` +cargo nextest run +``` +For checking the unit converage, you can run: +``` +cargo llvm-cov nextest +``` + +To test the CI/CD workflow locally, install `act` and run the following command: +``` +act -j build --container-architecture linux/amd64 -P ubuntu-latest=ubuntu:24.04 --reuse +``` \ No newline at end of file diff --git a/build-models.sh b/build-models.sh new file mode 100755 index 0000000..ec122d0 --- /dev/null +++ b/build-models.sh @@ -0,0 +1,77 @@ +# Exit immediately if a command exits with a non-zero status. +set -e + +# Ensure the OpenAPI Generator JAR file is set up +mkdir -p ~/bin/openapitools +OPENAPI_GENERATOR_JAR=~/bin/openapitools/openapi-generator-cli.jar +if [ ! -f "$OPENAPI_GENERATOR_JAR" ]; then + curl -L https://repo1.maven.org/maven2/org/openapitools/openapi-generator-cli/7.7.0/openapi-generator-cli-7.7.0.jar -o "$OPENAPI_GENERATOR_JAR" +fi + +get_git_repo_name() { + # Extract the URL of the remote "origin" + url=$(git config --get remote.origin.url) + + # Extract the repository name from the URL + repo_name=$(basename -s .git "$url") + + echo "$repo_name" +} + +SCRIPT_DIR="$(pwd)" + +generate_openapi_models() { + # Parameters + OPENAPI_SPEC_PATH="$1" + API_NAME="$2" + DESTINATION_DIR="$3" + + # Define the temporary output directory for the OpenAPI generator + TEMP_OUTPUT_DIR=$(mktemp -d) + + # Remove the temporary directory at the end of the script + trap 'rm -rf "$TEMP_OUTPUT_DIR"' EXIT + + # Run the OpenAPI generator CLI using the JAR file + java -jar "$OPENAPI_GENERATOR_JAR" generate -g rust \ + -i "$OPENAPI_SPEC_PATH" \ + -o "$TEMP_OUTPUT_DIR" \ + --additional-properties=useSingleRequestParameter=true + + # Check if the generation was successful + if [ $? -ne 0 ]; then + echo "OpenAPI generation failed. Check the verbose output for details." + exit 1 + fi + + # Remove the openapitools.json file + rm -f ./openapitools.json + + echo "TEMP_OUTPUT_DIR is $TEMP_OUTPUT_DIR" + + # Modify the import statements in each generated file + SED_RULE="s/use crate::models;/#![allow(unused_imports)]\n#![allow(clippy::empty_docs)]\nuse crate::$API_NAME::models;/" + for file in $(find "$TEMP_OUTPUT_DIR" -name '*.rs'); do + if [[ "$OSTYPE" == "darwin"* ]]; then + # macOS (BSD) sed syntax + sed -i '' "$SED_RULE" "$file" + else + # Linux (GNU) sed syntax + sed -i "$SED_RULE" "$file" + fi + done + + rm -rf "$DESTINATION_DIR/models" + mkdir -p "$DESTINATION_DIR" + cp -r "$TEMP_OUTPUT_DIR/src/models" "$DESTINATION_DIR" + + echo "OpenAPI generation complete. Models copied to $DESTINATION_DIR" +} + +generate_openapi_models \ + "https://raw.githubusercontent.com/ga4gh-discovery/ga4gh-service-info/develop/service-info.yaml" \ + "serviceinfo" "$SCRIPT_DIR/lib/src/serviceinfo/" + +generate_openapi_models \ + "https://raw.githubusercontent.com/ga4gh/task-execution-schemas/develop/openapi/task_execution_service.openapi.yaml" \ + "tes" "$SCRIPT_DIR/lib/src/tes/" \ No newline at end of file diff --git a/cli/Cargo.toml b/cli/Cargo.toml new file mode 100644 index 0000000..4dfcee6 --- /dev/null +++ b/cli/Cargo.toml @@ -0,0 +1,24 @@ +[package] +name = "cli" +description = """ +A cross-platform CLI tool written in Rust to use GA4GH-sdk +""" +repository = "https://github.com/elixir-cloud-aai/ga4gh-sdk/tree/main/cli" +version.workspace = true +authors.workspace = true +edition.workspace = true +readme.workspace = true +license-file.workspace = true + +[dependencies] +ga4gh-lib = { path = "../lib" } +clap = "3.0" +clap_complete = "3.0" +tokio = { version = "1", features = ["full"] } +serde_json = "^1.0" +tempfile = "3.2" +dirs = "5.0.1" + +[[bin]] +name = "cli" +path = "src/main.rs" \ No newline at end of file diff --git a/cli/src/main.rs b/cli/src/main.rs new file mode 100644 index 0000000..169c368 --- /dev/null +++ b/cli/src/main.rs @@ -0,0 +1,303 @@ +use clap::{arg, Command}; +use ga4gh_sdk::configuration::Configuration; +use ga4gh_sdk::tes::model::ListTasksParams; +use ga4gh_sdk::transport::Transport; +use std::collections::HashMap; +use std::error::Error; +use ga4gh_sdk::tes::{Task, TES}; +use ga4gh_sdk::tes::models::TesTask; +use ga4gh_sdk::test_utils::ensure_funnel_running; +use std::fs; +use std::path::Path; +use ga4gh_sdk::configuration::BasicAuth; +use std::fs::File; +use serde_json::Value; +use std::io::Read; + +/// # Examples +/// +/// To run the `create` command: +/// +/// ```sh +/// cargo run -- tes create '{ +/// "name": "Hello world", +/// "inputs": [{ +/// "url": "s3://funnel-bucket/hello.txt", +/// "path": "/inputs/hello.txt" +/// }], +/// "outputs": [{ +/// "url": "s3://funnel-bucket/output.txt", +/// "path": "/outputs/stdout" +/// }], +/// "executors": [{ +/// "image": "alpine", +/// "command": ["cat", "/inputs/hello.txt"], +/// "stdout": "/outputs/stdout" +/// }] +/// }' +/// ``` +/// +/// Or: +/// +/// ```sh +/// cargo run -- tes create './tests/sample.tes' +/// ``` +/// +/// To run the `list` command: +/// +/// ```sh +/// cargo run -- tes list 'name_prefix: None, state: None, tag_key: None, tag_value: None, page_size: None, page_token: None, view: FULL' +/// ``` +/// +/// ASSUME, cqgk5lj93m0311u6p530 is the id of a task created before +/// To run the `get` command: +/// +/// ```sh +/// cargo run -- tes get cqgk5lj93m0311u6p530 BASIC +/// ``` + +#[tokio::main] +async fn main() -> Result<(), Box> { + let cmd = Command::new("cli") + .bin_name("cli") + .version("1.0") + .about("CLI to manage tasks") + .subcommand_required(true) + .arg_required_else_help(true) + .subcommand( + Command::new("tes") + .about("TES subcommands") + .subcommand_required(true) + .arg_required_else_help(true) + .subcommand( + Command::new("create") + .about("Create a task") + .arg(arg!( "The task file to create")) + // .arg(arg!(--url "The URL for the task")) + .arg_required_else_help(true), + ) + + .subcommand( + Command::new("list") + .about("list all tasks") + .arg(arg!( "The parameters to get back")) + .arg_required_else_help(true), + ) + .subcommand( + Command::new("get") + .about("get task data") + .arg(arg!( "The id of the task which should be returned")) + .arg(arg!( "The view in which the task should be returned")) + .arg_required_else_help(true), + ) + .subcommand( + Command::new("status") + .about("get status of the task") + .arg(arg!( "The id of the task which should be returned")) + .arg_required_else_help(true), + ) + .subcommand( + Command::new("cancel") + .about("cancel the task") + .arg(arg!( "The id of the task which should be cancel")) + .arg_required_else_help(true), + ), + ); + + let matches = cmd.clone().get_matches(); + + match matches.subcommand() { + Some(("tes", sub)) => { + if let Some(("create", sub)) = sub.subcommand() { + let task_file = sub.value_of("TASK_FILE").unwrap(); + // let url = sub.value_of("url").unwrap(); + let path = Path::new(task_file); + if !path.exists() { + eprintln!("File does not exist: {:?}", path); + } + let task_json = match fs::read_to_string(path) { + Ok(contents) => contents, + Err(e) => { + eprintln!("Failed to read file: {}", e); + task_file.to_string() + }, + }; + let testask: TesTask = serde_json::from_str(&task_json) + .map_err(|e| format!("Failed to parse JSON: {}", e))?; + let mut config = load_configuration(); + if config.base_path == "localhost" { + let funnel_url = ensure_funnel_running().await; + config.set_base_path(&funnel_url); + } + match TES::new(&config).await { + Ok(tes) => { + let task = tes.create(testask).await; + println!("{:?}",task); + }, + Err(e) => { + println!("Error creating TES instance: {:?}", e); + return Err(e); + } + }; + } + if let Some(("list", sub)) = sub.subcommand() { + let params = sub.value_of("params").unwrap().to_string(); + + // Split the params string into key-value pairs and collect into a HashMap for easier access + let params_map: HashMap = params + .split(',') + .filter_map(|s| { + let mut parts = s.trim().splitn(2, ':'); + Some((parts.next()?.to_string(), parts.next()?.to_string())) + }) + .collect(); + println!("parameters are: {:?}",params_map); + + // Now, construct ListTasksParams from the parsed values + let parameters = ListTasksParams { + name_prefix: params_map.get("name_prefix").and_then(|s| if s == "None" { None } else { Some(s.to_string()) }), + state: params_map.get("state").and_then(|s| if s == "None" { None } else { Some(serde_json::from_str(s).expect("Invalid state")) }), + tag_key: None, // Example does not cover parsing Vec + tag_value: None, // Example does not cover parsing Vec + page_size: params_map.get("page_size").and_then(|s| if s == "None" { None } else { Some(s.parse().expect("Invalid page_size")) }), + page_token: params_map.get("page_token").and_then(|s| if s == "None" { None } else { Some(s.to_string()) }), + view: params_map.get("view").and_then(|s| if s == "None" { None } else { Some(s.to_string()) }), + }; + println!("parameters are: {:?}",parameters); + let mut config = load_configuration(); + if config.base_path == "localhost" { + let funnel_url = ensure_funnel_running().await; + config.set_base_path(&funnel_url); + } + match TES::new(&config).await { + Ok(tes) => { + let task = tes.list_tasks(Some(parameters)).await; + println!("{:?}",task); + }, + Err(e) => { + println!("Error creating TES instance: {:?}", e); + return Err(e); + } + }; + } + if let Some(("get", sub)) = sub.subcommand() { + let id = sub.value_of("id").unwrap(); + let view = sub.value_of("view").unwrap(); + + let mut config = load_configuration(); + if config.base_path == "localhost" { + let funnel_url = ensure_funnel_running().await; + config.set_base_path(&funnel_url); + } + + match TES::new(&config).await { + Ok(tes) => { + let task = tes.get(view, id).await; + println!("{:?}",task); + }, + Err(e) => { + println!("Error creating TES instance: {:?}", e); + return Err(e); + } + }; + } + if let Some(("status", sub)) = sub.subcommand() { + let id = sub.value_of("id").unwrap().to_string(); + + let mut config = load_configuration(); + if config.base_path == "localhost" { + let funnel_url = ensure_funnel_running().await; + config.set_base_path(&funnel_url); + } + let transport = Transport::new(&config); + let task = Task::new(id, transport); + match task.status().await { + Ok(status) => { + println!("The status is: {:?}",status); + }, + Err(e) => { + println!("Error creating Task instance: {:?}", e); + return Err(e); + } + }; + } + if let Some(("cancel", sub)) = sub.subcommand() { + let id = sub.value_of("id").unwrap().to_string(); + + let mut config = load_configuration(); + if config.base_path == "localhost" { + let funnel_url = ensure_funnel_running().await; + config.set_base_path(&funnel_url); + } + let transport = Transport::new(&config); + let task = Task::new(id, transport); + match task.cancel().await { + Ok(output) => { + println!("The new value is: {:?}",output); + }, + Err(e) => { + println!("Error creating Task instance: {:?}", e); + return Err(e); + } + }; + } + } + + _ => { + eprintln!("Error: Unrecognized command or option"); + std::process::exit(1); + } + } + Ok(()) +} + +/// Example `config.json` file: +/// +/// ```json +/// { +/// "base_path": "http://localhost:8000", +/// "user_agent": "username" +/// } +/// ``` + +fn read_configuration_from_file(file_path: &str) -> Result> { + let mut file = File::open(file_path)?; + let mut contents = String::new(); + file.read_to_string(&mut contents)?; + + let json_value: Value = serde_json::from_str(&contents)?; + + let base_path = json_value["base_path"].as_str().unwrap_or_default().to_string(); + let user_agent = json_value["user_agent"].as_str().map(|s| s.to_string()); + let basic_auth = json_value["basic_auth"].as_object().map(|auth| BasicAuth { + username: auth["username"].as_str().unwrap_or_default().to_string(), + password: Some(auth["password"].as_str().unwrap_or_default().to_string()), + }); + let oauth_access_token = json_value["oauth_access_token"].as_str().map(|s| s.to_string()); + + let config = Configuration::new(base_path, user_agent, basic_auth, oauth_access_token); + Ok(config) +} + +fn load_configuration() -> Configuration { + let config_file_path = dirs::home_dir().map(|path| path.join(".config/config.json")); + if let Some(path) = config_file_path { + if path.exists() { + if let Some(path_str) = path.to_str() { + match read_configuration_from_file(path_str) { + Ok(config) => {config}, + Err(_) => { + Configuration::default() + }, + } + } else { + Configuration::default() + } + + } else { + Configuration::default() + } + } else { + Configuration::default() + } +} diff --git a/lib/Cargo.toml b/lib/Cargo.toml new file mode 100644 index 0000000..5fcd198 --- /dev/null +++ b/lib/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "ga4gh-lib" +description = """ +Generic SDK and CLI for GA4GH API services +""" +repository = "https://github.com/elixir-cloud-aai/ga4gh-sdk/tree/main/lib" +version.workspace = true +authors.workspace = true +edition.workspace = true +readme.workspace = true +license-file.workspace = true +# rust-version = "1.74" + +[dependencies] +tokio = { version = "1", features = ["full"] } +serde = "^1.0" +serde_derive = "^1.0" +serde_json = "^1.0" +url = "^2.2" +uuid = { version = "^1.0", features = ["serde", "v4"] } +log = "0.4" +env_logger = "0.9" +once_cell = "1.8.0" + +[dependencies.reqwest] +version = "^0.11" +features = ["json", "multipart"] + +[dev-dependencies] +mockito = "0.31" +mockall = "0.10.2" +cargo-nextest = "0.9.30" + +[lib] +name = "ga4gh_sdk" +path = "src/lib.rs" diff --git a/lib/src/configuration.rs b/lib/src/configuration.rs new file mode 100644 index 0000000..c823b2c --- /dev/null +++ b/lib/src/configuration.rs @@ -0,0 +1,59 @@ +#[derive(Debug, Clone)] +pub struct Configuration { + pub base_path: String, + pub user_agent: Option, + pub basic_auth: Option, + pub oauth_access_token: Option, + pub bearer_access_token: Option, + pub api_key: Option, +} + +// Check whether defining BasicAuth works like this or not, else revert to the basic definition commented out +#[derive(Debug, Clone)] +pub struct BasicAuth { + pub username: String, + pub password: Option, +} +// pub type BasicAuth = (String, Option); + +#[derive(Debug, Clone)] +pub struct ApiKey { + pub prefix: Option, + pub key: String, +} + +impl Configuration { + pub fn new( + base_path: String, + user_agent: Option, + basic_auth: Option, + oauth_access_token: Option, + ) -> Self { + Configuration { + base_path, + user_agent, + basic_auth, + oauth_access_token, + bearer_access_token: None, + api_key: None, + } + } + + pub fn set_base_path(&mut self, base_path: &str) -> &mut Self { + self.base_path = base_path.to_string(); + self + } +} + +impl Default for Configuration { + fn default() -> Self { + Configuration { + base_path: "localhost".to_owned(), + user_agent: Some("GA4GH SDK".to_owned()), + basic_auth: None, + oauth_access_token: None, + bearer_access_token: None, + api_key: None, + } + } +} diff --git a/lib/src/lib.rs b/lib/src/lib.rs new file mode 100644 index 0000000..fc62146 --- /dev/null +++ b/lib/src/lib.rs @@ -0,0 +1,12 @@ +#[allow(unused_imports)] +#[macro_use] +extern crate serde_derive; + +// #[cfg(test)] +// mod test_utils; + +pub mod configuration; +pub mod serviceinfo; +pub mod tes; +pub mod transport; +pub mod test_utils; diff --git a/lib/src/serviceinfo/mod.rs b/lib/src/serviceinfo/mod.rs new file mode 100644 index 0000000..dbf532f --- /dev/null +++ b/lib/src/serviceinfo/mod.rs @@ -0,0 +1,62 @@ +pub mod models; +use crate::configuration::Configuration; +use crate::transport::Transport; + +#[derive(Clone)] +pub struct ServiceInfo { + transport: Transport, +} + +impl ServiceInfo { + pub fn new(config: &Configuration) -> Result> { + let transport = &Transport::new(config); + let instance = ServiceInfo { + transport: transport.clone(), + }; + Ok(instance) + } + + pub async fn get(&self) -> Result> { + let response = self.transport.get("/service-info", None).await; + match response { + Ok(response_body) => match serde_json::from_str::(&response_body) { + Ok(service) => Ok(service), + Err(e) => { + log::error!("Failed to deserialize response: {}", e); + Err(e.into()) + } + }, + Err(e) => { + log::error!("Error: {}", e); + Err(e) + } + } + } +} + +#[cfg(test)] +mod tests { + use crate::configuration::Configuration; + use crate::serviceinfo::ServiceInfo; + use crate::test_utils::{ensure_funnel_running, setup}; + use tokio; + + #[tokio::test] + async fn test_get_service_info_from_funnel() { + setup(); + let mut config = Configuration::default(); + let funnel_url = ensure_funnel_running().await; + config.set_base_path(&funnel_url); + let service_info = ServiceInfo::new(&config).unwrap(); + + // Call get_service_info and print the result + match service_info.get().await { + Ok(service) => { + println!("Service Info: {:?}", service); + } + Err(e) => { + println!("Failed to get service info: {}", e); + } + } + } +} diff --git a/lib/src/tes/mod.rs b/lib/src/tes/mod.rs new file mode 100644 index 0000000..0baad3d --- /dev/null +++ b/lib/src/tes/mod.rs @@ -0,0 +1,285 @@ +pub mod models; +pub mod model; +use crate::configuration::Configuration; +use crate::serviceinfo::models::Service; +use crate::serviceinfo::ServiceInfo; +use crate::tes::models::TesListTasksResponse; +use crate::tes::models::TesState; +use crate::tes::models::TesTask; +use crate::transport::Transport; +use crate::tes::model::ListTasksParams; +use serde_json; +use serde_json::from_str; +use serde_json::json; +use serde::Serialize; +use serde_json::Value; + +fn serialize_to_json(item: T) -> Value { + serde_json::to_value(&item).unwrap() +} + +pub fn urlencode>(s: T) -> String { + ::url::form_urlencoded::byte_serialize(s.as_ref().as_bytes()).collect() +} + +#[derive(Debug)] +pub struct Task { + id: String, + transport: Transport, +} + +impl Task { + pub fn new(id: String, transport: Transport) -> Self { + Task { id, transport } + } + + pub async fn status(&self) -> Result> { + let task_id = &self.id; + let view = "FULL"; + let url = format!("/tasks/{}?view={}", task_id, view); + // let params = [("view", view)]; + // let params_value = serde_json::json!(params); + // let response = self.transport.get(&url, Some(params_value)).await; + let response = self.transport.get(&url, None).await; + match response { + Ok(resp_str) => { + let task: TesTask = from_str(&resp_str)?; + Ok(task.state.unwrap()) + } + Err(e) => { + let err_msg = format!("HTTP request failed: {}", e); + eprintln!("{}", err_msg); + Err(Box::new(std::io::Error::new(std::io::ErrorKind::Other, err_msg))) + } + } + } + + pub async fn cancel(&self) -> Result> { + let id = &self.id; + let id = &urlencode(id); + let url = format!("/tasks/{}:cancel", id); + let response = self.transport.post(&url, None).await; + match response { + Ok(resp_str) => { + let parsed_json = serde_json::from_str::(&resp_str); + match parsed_json { + Ok(json) => Ok(json), + Err(e) => Err(format!("Failed to parse JSON: {}", e).into()), + } + } + Err(e) => Err(format!("HTTP request failed: {}", e).into()), + } + } +} +#[derive(Debug)] +pub struct TES { + #[allow(dead_code)] + config: Configuration, // not used yet + service: Result>, + transport: Transport, +} + +impl TES { + pub async fn new(config: &Configuration) -> Result> { + let transport = Transport::new(config); + let service_info = ServiceInfo::new(config)?; + + let resp = service_info.get().await; + + let instance = TES { + config: config.clone(), + transport, + service: resp, + }; + + instance.check()?; // Propagate the error if check() fails + Ok(instance) + } + + fn check(&self) -> Result<(), String> { + let resp = &self.service; + match resp.as_ref() { + Ok(service) if service.r#type.artifact == "tes" => Ok(()), + Ok(_) => Err("The endpoint is not an instance of TES".into()), + Err(_) => Err("Error accessing the service".into()), + } + } + + pub async fn create( + &self, + task: TesTask, /*, params: models::TesTask*/ + ) -> Result> { + // First, check if the service is of TES class + self.check().map_err(|e| { + log::error!("Service check failed: {}", e); + e + })?; + // todo: version in url based on serviceinfo or user config + let response = self + .transport + .post("/ga4gh/tes/v1/tasks", Some(json!(task))) + .await; + match response { + Ok(response_body) => { + let v: serde_json::Value = serde_json::from_str(&response_body)?; + + // Access the `id` field + let task_id = v + .get("id") + .and_then(|v| v.as_str()) + .unwrap_or_default() + .trim_matches('"') + .to_string(); + + let task = Task { + id: task_id, + transport: self.transport.clone(), + }; + Ok(task) + } + Err(e) => Err(Box::new(std::io::Error::new( + std::io::ErrorKind::Other, + format!("Failed to post task: {}", e), + ))), + } + } + + pub async fn get(&self, view: &str, id: &str) -> Result> { + let task_id = id; + let url = format!("/tasks/{}?view={}", task_id, view); + // let params = [("view", view)]; + // let params_value = serde_json::json!(params); + // let response = self.transport.get(&url, Some(params_value)).await; + let response = self.transport.get(&url, None).await; + match response { + Ok(resp_str) => { + let task: TesTask = from_str(&resp_str)?; + Ok(task) + } + Err(e) => Err(e), + } + } + pub async fn list_tasks( + &self, + params: Option, + ) -> Result> { + let params_value = params.map(serialize_to_json); + + // println!("{:?}",params_value); + // Make the request with or without parameters based on the presence of params + let response = if let Some(params_value) = params_value { + self.transport.get("/tasks", Some(params_value)).await + } else { + self.transport.get("/tasks", None).await + }; + + match response { + Ok(resp_str) => { + let task: TesListTasksResponse = from_str(&resp_str)?; + Ok(task) + } + Err(e) => { + eprintln!("HTTP request failed: {:?}", e); + Err(Box::new(std::io::Error::new( + std::io::ErrorKind::Other, + format!("HTTP request failed: {:?}", e), + ))) + } + } + } +} + +#[cfg(test)] +mod tests { + use crate::configuration::Configuration; + use crate::tes::models::TesTask; + use crate::tes::ListTasksParams; + use crate::tes::Task; + use crate::tes::TesState; + use crate::tes::TES; + use crate::test_utils::{ensure_funnel_running, setup}; + // use crate::tes::models::TesCreateTaskResponse; + + async fn create_task() -> Result<(Task, TES), Box> { + // setup(); – should be run once in the test function + let mut config = Configuration::default(); + let funnel_url = ensure_funnel_running().await; + config.set_base_path(&funnel_url); + println!("{:?}", funnel_url); + let tes = match TES::new(&config).await { + Ok(tes) => tes, + Err(e) => { + println!("Error creating TES instance: {:?}", e); + return Err(e); + } + }; + let file_path = "../tests/sample.tes".to_string(); + let task_json = std::fs::read_to_string(file_path).expect("Unable to read file"); + let task: TesTask = serde_json::from_str(&task_json).expect("JSON was not well-formatted"); + + let task = tes.create(task).await?; + Ok((task, tes)) + } + + #[tokio::test] + async fn test_task_create() { + setup(); + let (task, _tes) = create_task().await.expect("Failed to create task"); + assert!(!task.id.is_empty(), "Task ID should not be empty"); // double check if it's a correct assertion + } + + #[tokio::test] + async fn test_task_status() { + setup(); + + let (task, _tes) = create_task().await.expect("Failed to create task"); + assert!(!task.id.is_empty(), "Task ID should not be empty"); + + let status = task.status().await; + match status { + Ok(state) => { + assert!( + matches!(state, TesState::Initializing | TesState::Queued | TesState::Running), + "Unexpected state: {:?}", + state + ); + } + Err(err) => { + panic!("Task status returned an error: {:?}", err); + } + } + } + + #[tokio::test] + async fn test_cancel_task() { + setup(); + + let (task, _tes) = &create_task().await.expect("Failed to create task"); + assert!(!task.id.is_empty(), "Task ID should not be empty"); // double check if it's a correct assertion + + let cancel = task.cancel().await; + assert!(cancel.is_ok()); + } + + #[tokio::test] + async fn test_list_task() { + setup(); + + let (task, tes) = &create_task().await.expect("Failed to create task"); + assert!(!task.id.is_empty(), "Task ID should not be empty"); // double check if it's a correct assertion + + let params: ListTasksParams = ListTasksParams { + name_prefix: None, + state: None, + tag_key: None, + tag_value: None, + page_size: None, + page_token: None, + view: Some("BASIC".to_string()), + }; + + let list = tes.list_tasks(Some(params)).await; + assert!(list.is_ok()); + println!("{:?}", list); + } +} diff --git a/lib/src/tes/model.rs b/lib/src/tes/model.rs new file mode 100644 index 0000000..a67190d --- /dev/null +++ b/lib/src/tes/model.rs @@ -0,0 +1,27 @@ +use crate::tes::models; + +/// struct for passing parameters to the method [`list_tasks`] +#[derive(Serialize, Clone, Debug)] +pub struct ListTasksParams { + /// OPTIONAL. Filter the list to include tasks where the name matches this prefix. If unspecified, no task name filtering is done. + #[serde(skip_serializing_if = "Option::is_none")] + pub name_prefix: Option, + /// OPTIONAL. Filter tasks by state. If unspecified, no task state filtering is done. + #[serde(skip_serializing_if = "Option::is_none")] + pub state: Option, + /// OPTIONAL. Provide key tag to filter. The field tag_key is an array of key values, and will be zipped with an optional tag_value array. So the query: ``` ?tag_key=foo1&tag_value=bar1&tag_key=foo2&tag_value=bar2 ``` Should be constructed into the structure { \"foo1\" : \"bar1\", \"foo2\" : \"bar2\"} ``` ?tag_key=foo1 ``` Should be constructed into the structure {\"foo1\" : \"\"} If the tag_value is empty, it will be treated as matching any possible value. If a tag value is provided, both the tag's key and value must be exact matches for a task to be returned. Filter Tags Match? ---------------------------------------------------------------------- {\"foo\": \"bar\"} {\"foo\": \"bar\"} Yes {\"foo\": \"bar\"} {\"foo\": \"bat\"} No {\"foo\": \"\"} {\"foo\": \"\"} Yes {\"foo\": \"bar\", \"baz\": \"bat\"} {\"foo\": \"bar\", \"baz\": \"bat\"} Yes {\"foo\": \"bar\"} {\"foo\": \"bar\", \"baz\": \"bat\"} Yes {\"foo\": \"bar\", \"baz\": \"bat\"} {\"foo\": \"bar\"} No {\"foo\": \"\"} {\"foo\": \"bar\"} Yes {\"foo\": \"\"} {} No + #[serde(skip_serializing_if = "Option::is_none")] + pub tag_key: Option>, + /// OPTIONAL. The companion value field for tag_key + #[serde(skip_serializing_if = "Option::is_none")] + pub tag_value: Option>, + /// Optional number of tasks to return in one page. Must be less than 2048. Defaults to 256. + #[serde(skip_serializing_if = "Option::is_none")] + pub page_size: Option, + /// OPTIONAL. Page token is used to retrieve the next page of results. If unspecified, returns the first page of results. The value can be found in the `next_page_token` field of the last returned result of ListTasks + #[serde(skip_serializing_if = "Option::is_none")] + pub page_token: Option, + /// OPTIONAL. Affects the fields included in the returned Task messages. `MINIMAL`: Task message will include ONLY the fields: - `tesTask.Id` - `tesTask.State` `BASIC`: Task message will include all fields EXCEPT: - `tesTask.ExecutorLog.stdout` - `tesTask.ExecutorLog.stderr` - `tesInput.content` - `tesTaskLog.system_logs` `FULL`: Task message includes all fields. + #[serde(skip_serializing_if = "Option::is_none")] + pub view: Option, +} diff --git a/lib/src/test_utils.rs b/lib/src/test_utils.rs new file mode 100644 index 0000000..62763f5 --- /dev/null +++ b/lib/src/test_utils.rs @@ -0,0 +1,32 @@ +use std::env; +use std::process::Command; +use std::str; +use std::sync::Once; + +pub const FUNNEL_HOST: &str = "http://localhost"; +pub const FUNNEL_PORT: u16 = 8000; +pub static INIT: Once = Once::new(); + +pub fn setup() { + INIT.call_once(|| { + env::set_var("RUST_LOG", "debug"); + env_logger::init(); + }); +} + +pub async fn ensure_funnel_running() -> String { + let output = Command::new("sh") + .arg("-c") + .arg("ps aux | grep '[f]unnel server run'") + .output() + .expect("Failed to execute command"); + + let output_str = str::from_utf8(&output.stdout).unwrap(); + + if output_str.is_empty() { + panic!("Funnel is not running."); + } + + let funnel_url = format!("{}:{}", FUNNEL_HOST, FUNNEL_PORT); + funnel_url +} diff --git a/lib/src/transport.rs b/lib/src/transport.rs new file mode 100644 index 0000000..36389b9 --- /dev/null +++ b/lib/src/transport.rs @@ -0,0 +1,130 @@ +use crate::configuration::Configuration; +use log::error; +use reqwest::Client; +use serde_json::Value; +use std::error::Error; + +// note: could implement custom certs handling, such as in-TEE generated ephemerial certs +#[derive(Clone, Debug)] +pub struct Transport { + pub config: Configuration, + pub client: reqwest::Client, +} + +impl Transport { + pub fn new(config: &Configuration) -> Self { + Transport { + config: config.clone(), + client: Client::new(), + } + } + + async fn request( + &self, + method: reqwest::Method, + endpoint: &str, + data: Option, + params: Option, + ) -> Result> { + let full_url = format!("{}{}", self.config.base_path, endpoint); + let url = reqwest::Url::parse(&full_url).map_err(|_| { + error!("Invalid endpoint (shouldn't contain base url): {}", endpoint); + Box::new(std::io::Error::new(std::io::ErrorKind::InvalidInput, "Invalid endpoint")) as Box + })?; + + let mut request_builder = self.client.request(method, url).header( + reqwest::header::USER_AGENT, + self.config.user_agent.clone().unwrap_or_default(), + ); + + if let Some(ref params_value) = params { + // Validate or log params_value before setting it as query parameters + if params_value.is_object() { + request_builder = request_builder.query(params_value); + } else { + error!("params_value is not an object and cannot be used as query parameters: {:?}", params_value); + return Err(Box::new(std::io::Error::new(std::io::ErrorKind::InvalidInput, "params_value must be an object"))); + } + } + + if let Some(ref data) = data { + request_builder = request_builder.json(&data); + } + + let resp = request_builder.send().await.map_err(|e| { + eprintln!("HTTP request failed: {}", e); + e + })?; + + let status = resp.status(); + let content = resp.text().await.map_err(|e| format!("Failed to read response text: {}", e))?; + + if status.is_success() { + Ok(content) + } else { + Err(Box::new(std::io::Error::new( + std::io::ErrorKind::Other, + format!("Request failed with status: {}. Response: {}", status, content), + ))) + } + } + + pub async fn get( + &self, + endpoint: &str, + params: Option, + ) -> Result> { + self.request(reqwest::Method::GET, endpoint, None, params) + .await + } + + pub async fn post( + &self, + endpoint: &str, + data: Option, + ) -> Result> { + self.request(reqwest::Method::POST, endpoint, data, None) + .await + } + + pub async fn put(&self, endpoint: &str, data: Value) -> Result> { + self.request(reqwest::Method::PUT, endpoint, Some(data), None) + .await + } + + pub async fn delete(&self, endpoint: &str) -> Result> { + self.request(reqwest::Method::DELETE, endpoint, None, None) + .await + } + + // other HTTP methods can be added here +} + +#[cfg(test)] +mod tests { + use crate::configuration::Configuration; + use crate::test_utils::setup; + use crate::transport::Transport; + use mockito::mock; + + #[tokio::test] + async fn test_request() { + setup(); + let base_url = &mockito::server_url(); + // effectively no sense in testing various responses, as it's reqwest's responsibility + // we should test Transport's methods + let _m = mock("GET", "/test") + .with_status(200) + .with_header("content-type", "application/json") + .with_body(r#"{"message": "success"}"#) + .create(); + + let config = Configuration::new(base_url.clone(),None, None, None); + let transport = Transport::new(&config.clone()); + let response = transport.get("/test", None).await; + + assert!(response.is_ok()); + let body = response.unwrap(); + assert_eq!(body, r#"{"message": "success"}"#); + } +} diff --git a/nextest.toml b/nextest.toml new file mode 100644 index 0000000..02d1453 --- /dev/null +++ b/nextest.toml @@ -0,0 +1,15 @@ +[profile.default] +retries = 1 + +[script-pre-commands] +[[profile.default]] +commands = [ + { cmd = "sh", args = ["-c", "if ! ps aux | grep '[f]unnel server run'; then echo 'Funnel server is not running. Starting it now...'; export PATH=$PATH:~/go/bin; funnel server run --Server.HostName=localhost --Server.HTTPPort=8000 > funnel.log 2>&1 & fi"] }, + { cmd = "sh", args = ["-c", "while ! curl -s http://localhost:8000/healthz > /dev/null; do echo 'Waiting for Funnel server...'; sleep 1; done; echo 'Funnel server is running.'"] } +] + +[script-post-commands] +[[profile.default]] +commands = [ + # Add any post-test teardown commands here if needed +] diff --git a/run-tests.sh b/run-tests.sh new file mode 100755 index 0000000..4361c36 --- /dev/null +++ b/run-tests.sh @@ -0,0 +1,22 @@ +# Check if a "funnel" process is already running +if ! ps aux | grep '[f]unnel server run'; then + # If it's not running, start it + echo "Funnel server is not running. Starting it now..." + export PATH=$PATH:~/go/bin + funnel server run --Server.HostName=localhost --Server.HTTPPort=8000 > funnel.log 2>&1 & +else + echo "Funnel server is already running." +fi + +# Wait for the Funnel server to start +echo "Waiting for Funnel server to start..." +while ! curl -s http://localhost:8000/healthz > /dev/null +do + echo "Waiting for Funnel server..." + sleep 1 +done +echo "Funnel server is running." + +# Run the tests +RUST_BACKTRACE=1 RUST_LOG=debug cargo test + diff --git a/rustfmt.toml b/rustfmt.toml new file mode 100644 index 0000000..62437b6 --- /dev/null +++ b/rustfmt.toml @@ -0,0 +1,4 @@ +ignore = [ + "lib/serviceinfo/models", + "lib/tes/models", +] \ No newline at end of file diff --git a/tests/Readme.md b/tests/Readme.md new file mode 100644 index 0000000..21ba62f --- /dev/null +++ b/tests/Readme.md @@ -0,0 +1,7 @@ +# A folder for adding the files being used in unit tests + + +sample.tes: This is a sample file, taken from the [funnel docs](https://ohsu-comp-bio.github.io/funnel/docs/tasks/), and this file is being used in the file lib/src/tes/mod.rs + + +grape.tes: a sample file containing JSON task data for the GA4GH [Task Execution Service](https://github.com/ga4gh/task-execution-schemas), which can be used in the file lib/src/tes/mod.rs instead of sample.tes. Notably, it has placeholders like "${AWS_ACCESS_KEY_ID}" which is out of the standard and implies implementing a pre-processor, might be useful to note and implement in future as it avoids storing credentials in such .tes files diff --git a/tests/grape.tes b/tests/grape.tes new file mode 100644 index 0000000..2cceaf6 --- /dev/null +++ b/tests/grape.tes @@ -0,0 +1,85 @@ +{ + "name": "GRAPE", + "resources": { + "disk_gb": 200 + }, + "volumes": [ + "/vol/a/" + ], + "executors": [ + { + "image": "amazon/aws-cli", + "command": [ + "aws", + "s3", + "cp", + "${INPUT}", + "/vol/a/input.vcf.gz" + ], + "env": { + "AWS_ACCESS_KEY_ID": "${AWS_ACCESS_KEY_ID}", + "AWS_SECRET_ACCESS_KEY": "${AWS_SECRET_ACCESS_KEY}", + "AWS_REGION": "${AWS_REGION}" + } + }, + { + "image": "genxnetwork/grape", + "command": [ + "python", + "launcher.py", + "reference", + "--use-bundle", + "--ref-directory", + "/vol/a/media/ref", + "--real-run" + ] + }, + { + "image": "genxnetwork/grape", + "command": [ + "python", + "launcher.py", + "preprocess", + "--ref-directory", + "/vol/a/media/ref", + "--vcf-file", + "/vol/a/input.vcf.gz", + "--directory", + "/vol/a/media/data", + "--assembly", + "hg37", + "--real-run" + ] + }, + { + "image": "genxnetwork/grape", + "command": [ + "python", + "launcher.py", + "find", + "--flow", + "ibis", + "--ref-directory", + "/vol/a/media/ref", + "--directory", + "/vol/a/media/data", + "--real-run" + ] + }, + { + "image": "amazon/aws-cli", + "command": [ + "aws", + "s3", + "cp", + "/vol/a/media/data/results/relatives.tsv", + "${OUTPUT}" + ], + "env": { + "AWS_ACCESS_KEY_ID": "${AWS_ACCESS_KEY_ID}", + "AWS_SECRET_ACCESS_KEY": "${AWS_SECRET_ACCESS_KEY}", + "AWS_REGION": "${AWS_REGION}" + } + } + ] +} \ No newline at end of file diff --git a/tests/sample.tes b/tests/sample.tes new file mode 100644 index 0000000..e663c3a --- /dev/null +++ b/tests/sample.tes @@ -0,0 +1,16 @@ +{ + "name": "Hello world", + "inputs": [{ + "url": "s3://funnel-bucket/hello.txt", + "path": "/inputs/hello.txt" + }], + "outputs": [{ + "url": "s3://funnel-bucket/output.txt", + "path": "/outputs/stdout" + }], + "executors": [{ + "image": "alpine", + "command": ["cat", "/inputs/hello.txt"], + "stdout": "/outputs/stdout" + }] +}