From 27bb2fe9a2766d36573d5d7f85d6fea3877b351b Mon Sep 17 00:00:00 2001 From: DE MERINGO Olivier Date: Wed, 26 Jul 2023 16:45:01 +0200 Subject: [PATCH] wip: refactor returned results to expose an OpenAPI spec. --- Cargo.lock | 126 +++++++++++++++++++++++ Cargo.toml | 1 + cloud-scanner-cli/Cargo.toml | 13 ++- cloud-scanner-cli/src/cloud_resource.rs | 8 +- cloud-scanner-cli/src/impact_provider.rs | 8 +- cloud-scanner-cli/src/lib.rs | 39 +++++-- cloud-scanner-cli/src/metric_server.rs | 76 +++++++++++++- cloud-scanner-cli/src/model.rs | 25 ++++- cloud-scanner-cli/src/usage_location.rs | 4 +- 9 files changed, 279 insertions(+), 21 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b3cd17ce..d16f95db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -731,6 +731,7 @@ dependencies = [ "pkg-version", "prometheus-client", "rocket", + "rocket_okapi", "serde", "serde_derive", "serde_json", @@ -804,6 +805,41 @@ dependencies = [ "typenum", ] +[[package]] +name = "darling" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a01d95850c592940db9b8194bc39f4bc0e89dee5c4265e4b1807c34a9aba453c" +dependencies = [ + "darling_core", + "darling_macro", +] + +[[package]] +name = "darling_core" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "859d65a907b6852c9361e3185c862aae7fafd2887876799fa55f5f99dc40d610" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 1.0.109", +] + +[[package]] +name = "darling_macro" +version = "0.13.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c972679f83bdf9c42bd905396b6c3588a843a17f0f16dfcfa3e2c5d57441835" +dependencies = [ + "darling_core", + "quote", + "syn 1.0.109", +] + [[package]] name = "devise" version = "0.4.1" @@ -854,6 +890,12 @@ version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dcbb2bf8e87535c23f7a8a321e364ce21462d0ff10cb6407820e8e96dfff6653" +[[package]] +name = "dyn-clone" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "304e6508efa593091e97a9abbc10f90aa7ca635b6d2784feff3c89d41dd12272" + [[package]] name = "either" version = "1.9.0" @@ -1271,6 +1313,12 @@ dependencies = [ "cc", ] +[[package]] +name = "ident_case" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" + [[package]] name = "idna" version = "0.4.0" @@ -1594,6 +1642,18 @@ dependencies = [ "memchr", ] +[[package]] +name = "okapi" +version = "0.7.0-rc.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ce66b6366e049880a35c378123fddb630b1a1a3c37fa1ca70caaf4a09f6e2893" +dependencies = [ + "log", + "schemars", + "serde", + "serde_json", +] + [[package]] name = "once_cell" version = "1.18.0" @@ -1983,6 +2043,7 @@ dependencies = [ "rocket_codegen", "rocket_http", "serde", + "serde_json", "state", "tempfile", "time 0.3.23", @@ -2037,6 +2098,35 @@ dependencies = [ "uncased", ] +[[package]] +name = "rocket_okapi" +version = "0.8.0-rc.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "742098674565c8f0c35c77444f90344aafedebb71cfee9cdbf0185acc6b9cdb7" +dependencies = [ + "either", + "log", + "okapi", + "rocket", + "rocket_okapi_codegen", + "schemars", + "serde", + "serde_json", +] + +[[package]] +name = "rocket_okapi_codegen" +version = "0.8.0-rc.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c43f8edc57d88750a220b0ec1870a36c1106204ec99cc35131b49de3b954a4a" +dependencies = [ + "darling", + "proc-macro2", + "quote", + "rocket_http", + "syn 1.0.109", +] + [[package]] name = "rustc-demangle" version = "0.1.23" @@ -2141,6 +2231,31 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "schemars" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02c613288622e5f0c3fdc5dbd4db1c5fbe752746b1d1a56a0630b78fd00de44f" +dependencies = [ + "dyn-clone", + "indexmap 1.9.3", + "schemars_derive", + "serde", + "serde_json", +] + +[[package]] +name = "schemars_derive" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109da1e6b197438deb6db99952990c7f959572794b80ff93707d55a232545e7c" +dependencies = [ + "proc-macro2", + "quote", + "serde_derive_internals", + "syn 1.0.109", +] + [[package]] name = "scoped-tls" version = "1.0.1" @@ -2212,6 +2327,17 @@ dependencies = [ "syn 2.0.27", ] +[[package]] +name = "serde_derive_internals" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85bf8229e7920a9f636479437026331ce11aa132b4dde37d121944a44d6e5f3c" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + [[package]] name = "serde_json" version = "1.0.103" diff --git a/Cargo.toml b/Cargo.toml index eaaaddc8..ebd521d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,4 +1,5 @@ [workspace] +resolver = "2" default-members = [ "cloud-scanner-cli", diff --git a/cloud-scanner-cli/Cargo.toml b/cloud-scanner-cli/Cargo.toml index 55178273..fbc753fb 100644 --- a/cloud-scanner-cli/Cargo.toml +++ b/cloud-scanner-cli/Cargo.toml @@ -3,6 +3,7 @@ authors = ["boavizta.org", "Olivier de Meringo "] edition = "2021" name = "cloud-scanner-cli" version = "0.3.0-alpha3" + [dependencies] aws-types = "0.55.3" chrono = "^0.4" @@ -15,10 +16,20 @@ serde = "^1.0" serde_derive = "^1.0" serde_json = "^1.0" anyhow = "1.0.65" -rocket = "0.5.0-rc.2" + async-trait = "0.1.58" assert-json-diff = "2.0.2" +rocket = { version ="0.5.0-rc.3", default-features = false, features = ["json"]} +rocket_okapi = { version="0.8.0-rc.3", features = ["swagger", "rapidoc"]} + +# schemars = { version = "0.8.10" } +# okapi = { version = "0.7", features = ["derive_json_schema"] } + + +# okapi = { version = "0.4", features = ["derive_json_schema"] } + + [dependencies.boavizta_api_sdk] version = "0.3.0-alpha2" # path = "../../boaviztapi-sdk-rust" diff --git a/cloud-scanner-cli/src/cloud_resource.rs b/cloud-scanner-cli/src/cloud_resource.rs index 4eb8b92e..7a35fbb6 100644 --- a/cloud-scanner-cli/src/cloud_resource.rs +++ b/cloud-scanner-cli/src/cloud_resource.rs @@ -1,10 +1,12 @@ use crate::UsageLocation; //use anyhow::{Context, Result}; +use rocket_okapi::okapi::schemars; +use rocket_okapi::okapi::schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::{collections::HashMap, fmt}; /// A cloud resource (could be an instance, function or any other resource) -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] pub struct CloudResource { pub provider: String, pub id: String, @@ -22,14 +24,14 @@ impl fmt::Display for CloudResource { } /// Usage of a cloud resource -#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, Default, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct CloudResourceUsage { pub average_cpu_load: f64, pub usage_duration_seconds: u32, } /// A tag (just a mandatory key + optional value) -#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize)] +#[derive(Clone, Debug, Eq, PartialEq, Serialize, Deserialize, JsonSchema)] pub struct CloudResourceTag { pub key: String, pub value: Option, diff --git a/cloud-scanner-cli/src/impact_provider.rs b/cloud-scanner-cli/src/impact_provider.rs index 10b2441d..aae2322f 100644 --- a/cloud-scanner-cli/src/impact_provider.rs +++ b/cloud-scanner-cli/src/impact_provider.rs @@ -8,6 +8,8 @@ use crate::cloud_resource::*; use anyhow::Result; use async_trait::async_trait; +use rocket_okapi::okapi::schemars; +use rocket_okapi::okapi::schemars::JsonSchema; use serde::{Deserialize, Serialize}; /// A ImpactProvider trait that yu should implement for a specific impact API @@ -24,7 +26,7 @@ pub trait ImpactProvider { ) -> Result>; } -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] pub struct CloudResourceWithImpacts { pub cloud_resource: CloudResource, /// The impacts @@ -34,7 +36,7 @@ pub struct CloudResourceWithImpacts { } /// Impacts of an individual resource -#[derive(Clone, Debug, Default, Serialize, Deserialize)] +#[derive(Clone, Debug, Default, Serialize, Deserialize, JsonSchema)] pub struct ResourceImpacts { pub adp_manufacture_kgsbeq: f64, pub adp_use_kgsbeq: f64, @@ -45,7 +47,7 @@ pub struct ResourceImpacts { } /// The aggregated impacts and meta data about the scan results -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] pub struct ImpactsSummary { pub number_of_instances_total: u32, pub number_of_instances_assessed: u32, diff --git a/cloud-scanner-cli/src/lib.rs b/cloud-scanner-cli/src/lib.rs index 894852b0..730f5d44 100644 --- a/cloud-scanner-cli/src/lib.rs +++ b/cloud-scanner-cli/src/lib.rs @@ -3,7 +3,7 @@ //! A command line application that performs inventory of your cloud account and combines it with Boavizta API to return an estimation of its environmental impact. //! -use crate::model::ExecutionStatistics; +use crate::model::{ExecutionStatistics, ResourcesWithImpacts}; use crate::usage_location::*; use aws_inventory::*; use boavizta_api_v1::*; @@ -18,6 +18,7 @@ extern crate rocket; #[macro_use] extern crate log; +use model::Inventory; use pkg_version::*; use std::time::{Duration, Instant}; pub mod aws_inventory; @@ -37,20 +38,20 @@ async fn standard_scan( tags: &[String], aws_region: &str, api_url: &str, -) -> Result> { +) -> Result { let start = Instant::now(); let inventory: AwsInventory = AwsInventory::new(aws_region).await; let cloud_resources: Vec = inventory .list_resources(tags) .await - .context("Cannot perform resouces inventory")?; + .context("Cannot perform resources inventory")?; let inventory_duration = start.elapsed(); let impact_start = Instant::now(); let api: BoaviztaApiV1 = BoaviztaApiV1::new(api_url); - let res = api + let vec = api .get_impacts(cloud_resources, hours_use_time) .await .context("Failure while retrieving impacts")?; @@ -66,10 +67,15 @@ async fn standard_scan( info!("{}", stats); + let res: ResourcesWithImpacts = ResourcesWithImpacts { + impacts: vec, + execution_statistics: stats, + }; + Ok(res) } -/// Returns default impacts as json +/// Returns default impacts as json string pub async fn get_default_impacts_as_json_string( hours_use_time: &f32, tags: &[String], @@ -98,7 +104,7 @@ pub async fn get_default_impacts_as_metrics( let summary: ImpactsSummary = ImpactsSummary::new( String::from(aws_region), usage_location.iso_country_code, - instances_with_impacts, + instances_with_impacts.impacts, (*hours_use_time).into(), ); debug!("Summary: {:#?}", summary); @@ -153,6 +159,27 @@ pub async fn get_inventory_as_json(tags: &[String], aws_region: &str) -> Result< serde_json::to_string(&cloud_resources).context("Cannot format inventory as json") } +pub async fn get_inventory(tags: &[String], aws_region: &str) -> Result { + let start = Instant::now(); + let aws_inventory: AwsInventory = AwsInventory::new(aws_region).await; + let cloud_resources: Vec = aws_inventory + .list_resources(tags) + .await + .context("Cannot perform inventory.")?; + let stats = ExecutionStatistics { + inventory_duration: start.elapsed(), + impact_duration: Duration::from_millis(0), + total_duration: start.elapsed(), + }; + warn!("{:?}", stats); + + let inventory = Inventory { + resources: cloud_resources, + execution_statistics: stats, + }; + Ok(inventory) +} + /// List instances and metadata to standard output pub async fn show_inventory(tags: &[String], aws_region: &str) -> Result<()> { let json_inventory: String = get_inventory_as_json(tags, aws_region).await?; diff --git a/cloud-scanner-cli/src/metric_server.rs b/cloud-scanner-cli/src/metric_server.rs index 2f51a536..e7cd6772 100644 --- a/cloud-scanner-cli/src/metric_server.rs +++ b/cloud-scanner-cli/src/metric_server.rs @@ -2,6 +2,16 @@ use rocket::State; +use rocket::form::FromForm; +use rocket::{get, post, serde::json::Json}; +use rocket_okapi::okapi::schemars; +use rocket_okapi::okapi::schemars::JsonSchema; +use rocket_okapi::settings::UrlObject; +use rocket_okapi::{openapi, openapi_get_routes, rapidoc::*, swagger_ui::*}; +use serde::{Deserialize, Serialize}; + +use crate::model::{Inventory, ResourcesWithImpacts}; + /// Configuration for the metric server pub struct Config { pub boavizta_url: String, @@ -10,7 +20,17 @@ pub struct Config { /// Start the metric server pub async fn run(config: Config) -> Result<(), rocket::Error> { let _rocket = rocket::build() - .mount("/", routes![index, metrics, inventory]) + .mount( + "/", + openapi_get_routes![index, metrics, inventory, inventorynew, impacts], + ) + .mount( + "/swagger-ui/", + make_swagger_ui(&SwaggerUIConfig { + url: "../openapi.json".to_owned(), + ..Default::default() + }), + ) .manage(config) .launch() .await?; @@ -18,16 +38,18 @@ pub async fn run(config: Config) -> Result<(), rocket::Error> { } /// Just display help +#[openapi(skip)] #[get("/")] fn index(config: &State) -> String { warn!("Getting request on /"); let version: String = crate::get_version(); - format!("Cloud scanner metric server {} is running.\n\nUsing Boavizta API at: {}.\nMetrics are exposed on /metrics path and require passing a **region** in query string.\n e.g. http://localhost:8000/metrics?aws_region=eu-west-3", version, config.boavizta_url) + format!("Cloud scanner metric server {} is running.\n\nUsing Boavizta API at: {}.\nMetrics are exposed on /metrics path and require passing a **region** in query string.\n e.g. http://localhost:8000/metrics?aws_region=eu-west-3 \n See also /swagger-ui .", version, config.boavizta_url) } /// Returns the metrics /// Region is mandatory, tags are optional /// Example query: http://localhost:8000/metrics?aws_region=eu-west-3&filter_tag=Name=boatest&filter_tag=OtherTag=other-value +#[openapi(skip)] #[get("/metrics?&")] async fn metrics(config: &State, aws_region: &str, filter_tags: Vec) -> String { warn!("Getting something on /metrics"); @@ -47,9 +69,55 @@ async fn metrics(config: &State, aws_region: &str, filter_tags: Vec&")] async fn inventory(_config: &State, aws_region: &str, filter_tags: Vec) -> String { warn!("Getting something on /inventory"); warn!("Filtering on tags {:?}", filter_tags); - crate::get_inventory_as_json(&filter_tags, aws_region).await.unwrap() -} \ No newline at end of file + crate::get_inventory_as_json(&filter_tags, aws_region) + .await + .unwrap() +} + +/// Returns the inventory as json +/// Region is mandatory, tags are optional +/// Example query: http://localhost:8000/inventorynew?aws_region=eu-west-3&filter_tag=Name=boatest&filter_tag=OtherTag=other-value +#[openapi(tag = "inventory")] +#[get("/inventorynew?&")] +async fn inventorynew( + _config: &State, + aws_region: &str, + filter_tags: Vec, +) -> Json { + warn!("Getting something on /inventorynew"); + warn!("Filtering on tags {:?}", filter_tags); + Json( + crate::get_inventory(&filter_tags, aws_region) + .await + .unwrap(), + ) +} + +/// Returns the impacts as json +/// Region is mandatory, tags are optional +/// Example query: http://localhost:8000/impacts?aws_region=eu-west-3&filter_tag=Name=boatest&filter_tag=OtherTag=other-value +#[openapi(tag = "impacts")] +#[get("/impacts?&")] +async fn impacts( + _config: &State, + aws_region: &str, + filter_tags: Vec, +) -> Json { + warn!("Getting something on /impacts"); + let hours_use_time: f32 = 1.0; + warn!("Filtering on tags {:?}", filter_tags); + let res = crate::standard_scan( + &hours_use_time, + &filter_tags, + aws_region, + &_config.boavizta_url, + ) + .await + .unwrap(); + Json(res) +} diff --git a/cloud-scanner-cli/src/model.rs b/cloud-scanner-cli/src/model.rs index 8391100f..a3f83d78 100644 --- a/cloud-scanner-cli/src/model.rs +++ b/cloud-scanner-cli/src/model.rs @@ -1,10 +1,13 @@ -use std::time::Duration; - +use rocket_okapi::okapi::schemars; +use rocket_okapi::okapi::schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::fmt; +use std::time::Duration; + +use crate::{cloud_resource::CloudResource, impact_provider::CloudResourceWithImpacts}; /// Statistics about program execution -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] pub struct ExecutionStatistics { pub inventory_duration: Duration, pub impact_duration: Duration, @@ -16,3 +19,19 @@ impl fmt::Display for ExecutionStatistics { write!(f, "{:?}", self) } } + +/// Inventory +#[derive(Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct Inventory { + pub resources: Vec, + pub execution_statistics: ExecutionStatistics, +} + +/// Impacts results +#[derive(Serialize, Deserialize, JsonSchema)] +#[serde(rename_all = "camelCase")] +pub struct ResourcesWithImpacts { + pub impacts: Vec, + pub execution_statistics: ExecutionStatistics, +} diff --git a/cloud-scanner-cli/src/usage_location.rs b/cloud-scanner-cli/src/usage_location.rs index 8f1bc726..4bcedfe0 100644 --- a/cloud-scanner-cli/src/usage_location.rs +++ b/cloud-scanner-cli/src/usage_location.rs @@ -1,8 +1,10 @@ use isocountry::CountryCode; +use rocket_okapi::okapi::schemars; +use rocket_okapi::okapi::schemars::JsonSchema; use serde::{Deserialize, Serialize}; /// TODO! the usage location should be part of the cloud_inventory model (region names are tied to a specific cloud provider) -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] pub struct UsageLocation { pub aws_region: String, /// The 3-letters ISO country code corresponding to the country of the aws_region