From 3c48f8c0efb1ee83f84c6bf04527d171441af85e Mon Sep 17 00:00:00 2001 From: olivier de Meringo Date: Thu, 11 Jul 2024 19:13:40 +0200 Subject: [PATCH 1/2] feat: add CLI option to get the json schema of inventory file. --- CHANGELOG.md | 1 + cloud-scanner-cli/src/inventory_exporter.rs | 29 ++ cloud-scanner-cli/src/main.rs | 22 +- cloud-scanner-cli/src/model.rs | 10 +- .../test-data/INVENTORY_JSON_SCHEMA.json | 305 ++++++++++++++++++ 5 files changed, 359 insertions(+), 8 deletions(-) create mode 100644 cloud-scanner-cli/test-data/INVENTORY_JSON_SCHEMA.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 382e21d2..699221d2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 _This paragraph may describe WIP/unreleased features. They are merged to main branch but not tagged._ - [Doc: reference Boavizta methodology paper. · Issue #552 · Boavizta/cloud-scanner](https://github.com/Boavizta/cloud-scanner/issues/552) +- [Expose the json schema of the inventory format · Issue #558 · Boavizta/cloud-scanner](https://github.com/Boavizta/cloud-scanner/issues/558). Use `cargo run inventory --print-json-schema` with CLI to get the schema on stdout. ## [3.0.1]-2024-06-19 diff --git a/cloud-scanner-cli/src/inventory_exporter.rs b/cloud-scanner-cli/src/inventory_exporter.rs index 962daa3e..ec23a183 100644 --- a/cloud-scanner-cli/src/inventory_exporter.rs +++ b/cloud-scanner-cli/src/inventory_exporter.rs @@ -1,4 +1,5 @@ use anyhow::Context; +use schemars::schema_for; use crate::model::Inventory; @@ -13,3 +14,31 @@ pub async fn print_inventory(inventory: &Inventory) -> anyhow::Result<()> { println!("{}", json_inventory); Ok(()) } + +/// Returns the json schema of an inventory as String +pub fn get_inventory_schema() -> anyhow::Result { + let schema = schema_for!(Inventory); + let st = serde_json::to_string_pretty(&schema)?; + Ok(st) +} + +/// Print inventory schema on stdout +pub fn print_inventory_schema() -> anyhow::Result<()> { + let s = get_inventory_schema()?; + println!("{}", s); + Ok(()) +} + +#[cfg(test)] +mod tests { + use crate::inventory_exporter::get_inventory_schema; + const INVENTORY_JSON_SCHEMA: &str = include_str!("../test-data/INVENTORY_JSON_SCHEMA.json"); + + #[test] + pub fn generate_inventory_schema() { + let s = get_inventory_schema().unwrap(); + println!("{}", s); + + assert_eq!(s, INVENTORY_JSON_SCHEMA, "schema do not match"); + } +} diff --git a/cloud-scanner-cli/src/main.rs b/cloud-scanner-cli/src/main.rs index 82bd8222..b95a0b02 100644 --- a/cloud-scanner-cli/src/main.rs +++ b/cloud-scanner-cli/src/main.rs @@ -72,6 +72,10 @@ enum SubCommand { #[arg(long, short = 'b', action)] /// Experimental feature: include block storage in the inventory include_block_storage: bool, + + /// Print the json schema of the inventory (instead of performing inventory) + #[arg(short = 's', long)] + print_json_schema: bool, }, /// Run as a standalone server. /// Access metrics (e.g. http://localhost:8000/metrics?aws_region=eu-west-3), inventory or impacts (see http://localhost:8000/swagger-ui) @@ -168,12 +172,20 @@ async fn main() -> Result<()> { } SubCommand::Inventory { include_block_storage, + print_json_schema, } => { - info!("Using filter tags {:?}", &args.filter_tags); - let inventory = - cloud_scanner_cli::get_inventory(&args.filter_tags, ®ion, include_block_storage) - .await?; - print_inventory(&inventory).await?; + if print_json_schema { + cloud_scanner_cli::inventory_exporter::print_inventory_schema()?; + } else { + info!("Using filter tags {:?}", &args.filter_tags); + let inventory = cloud_scanner_cli::get_inventory( + &args.filter_tags, + ®ion, + include_block_storage, + ) + .await?; + print_inventory(&inventory).await?; + } } SubCommand::Serve {} => cloud_scanner_cli::serve_metrics(&api_url).await?, } diff --git a/cloud-scanner-cli/src/model.rs b/cloud-scanner-cli/src/model.rs index 1def4400..2d2c48e0 100644 --- a/cloud-scanner-cli/src/model.rs +++ b/cloud-scanner-cli/src/model.rs @@ -12,7 +12,7 @@ use std::{fmt, fs}; use crate::impact_provider::CloudResourceWithImpacts; use crate::usage_location::UsageLocation; -/// Statistics about program execution +/// Statistics about program execution #[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "snake_case")] pub struct ExecutionStatistics { @@ -27,7 +27,7 @@ impl fmt::Display for ExecutionStatistics { } } -/// Inventory: a list of resources +/// A list of cloud resources and metadata that describes the inventory itself #[derive(Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "snake_case")] pub struct Inventory { @@ -39,10 +39,13 @@ pub struct Inventory { #[derive(Clone, Serialize, Deserialize, JsonSchema)] #[serde(rename_all = "snake_case")] pub struct InventoryMetadata { + /// The date when the inventory was generated pub inventory_date: Option>, + /// A free text description of the inventory pub description: Option, /// The version of the cloud scanner that generated the inventory pub cloud_scanner_version: Option, + /// Statistics about program execution pub execution_statistics: Option, } @@ -67,11 +70,12 @@ pub struct EstimatedInventory { pub execution_statistics: Option, } -/// A cloud resource (could be an instance, function or any other resource) +/// A cloud resource (could be an instance, block storage or any other resource) #[derive(Clone, Debug, Serialize, Deserialize, JsonSchema)] pub struct CloudResource { pub provider: CloudProvider, pub id: String, + /// The location where cloud resources are running. pub location: UsageLocation, pub resource_details: ResourceDetails, pub tags: Vec, diff --git a/cloud-scanner-cli/test-data/INVENTORY_JSON_SCHEMA.json b/cloud-scanner-cli/test-data/INVENTORY_JSON_SCHEMA.json new file mode 100644 index 00000000..42b303bd --- /dev/null +++ b/cloud-scanner-cli/test-data/INVENTORY_JSON_SCHEMA.json @@ -0,0 +1,305 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Inventory", + "description": "A list of cloud resources and metadata that describes the inventory itself", + "type": "object", + "required": [ + "metadata", + "resources" + ], + "properties": { + "metadata": { + "$ref": "#/definitions/InventoryMetadata" + }, + "resources": { + "type": "array", + "items": { + "$ref": "#/definitions/CloudResource" + } + } + }, + "definitions": { + "InventoryMetadata": { + "description": "Details about the inventory", + "type": "object", + "properties": { + "inventory_date": { + "description": "The date when the inventory was generated", + "type": [ + "string", + "null" + ], + "format": "date-time" + }, + "description": { + "description": "A free text description of the inventory", + "type": [ + "string", + "null" + ] + }, + "cloud_scanner_version": { + "description": "The version of the cloud scanner that generated the inventory", + "type": [ + "string", + "null" + ] + }, + "execution_statistics": { + "description": "Statistics about program execution", + "anyOf": [ + { + "$ref": "#/definitions/ExecutionStatistics" + }, + { + "type": "null" + } + ] + } + } + }, + "ExecutionStatistics": { + "description": "Statistics about program execution", + "type": "object", + "required": [ + "impact_estimation_duration", + "inventory_duration", + "total_duration" + ], + "properties": { + "inventory_duration": { + "$ref": "#/definitions/Duration" + }, + "impact_estimation_duration": { + "$ref": "#/definitions/Duration" + }, + "total_duration": { + "$ref": "#/definitions/Duration" + } + } + }, + "Duration": { + "type": "object", + "required": [ + "nanos", + "secs" + ], + "properties": { + "secs": { + "type": "integer", + "format": "uint64", + "minimum": 0.0 + }, + "nanos": { + "type": "integer", + "format": "uint32", + "minimum": 0.0 + } + } + }, + "CloudResource": { + "description": "A cloud resource (could be an instance, block storage or any other resource)", + "type": "object", + "required": [ + "id", + "location", + "provider", + "resource_details", + "tags" + ], + "properties": { + "provider": { + "$ref": "#/definitions/CloudProvider" + }, + "id": { + "type": "string" + }, + "location": { + "description": "The location where cloud resources are running.", + "allOf": [ + { + "$ref": "#/definitions/UsageLocation" + } + ] + }, + "resource_details": { + "$ref": "#/definitions/ResourceDetails" + }, + "tags": { + "type": "array", + "items": { + "$ref": "#/definitions/CloudResourceTag" + } + } + } + }, + "CloudProvider": { + "type": "string", + "enum": [ + "AWS", + "OVH" + ] + }, + "UsageLocation": { + "description": "The location where cloud resources are running.\n\nTODO! the usage location should be abstracted and vendor specific implementation should be part of the cloud_provider model (region names are tied to a specific cloud provider)", + "type": "object", + "required": [ + "aws_region", + "iso_country_code" + ], + "properties": { + "aws_region": { + "description": "The AWS region (like eu-west-1)", + "type": "string" + }, + "iso_country_code": { + "description": "The 3-letters ISO country code corresponding to the country of the aws_region", + "type": "string" + } + } + }, + "ResourceDetails": { + "oneOf": [ + { + "type": "string", + "enum": [ + "object_storage" + ] + }, + { + "type": "object", + "required": [ + "instance" + ], + "properties": { + "instance": { + "type": "object", + "required": [ + "instance_type" + ], + "properties": { + "instance_type": { + "type": "string" + }, + "usage": { + "anyOf": [ + { + "$ref": "#/definitions/InstanceUsage" + }, + { + "type": "null" + } + ] + } + } + } + }, + "additionalProperties": false + }, + { + "type": "object", + "required": [ + "block_storage" + ], + "properties": { + "block_storage": { + "type": "object", + "required": [ + "storage_type" + ], + "properties": { + "storage_type": { + "type": "string" + }, + "usage": { + "anyOf": [ + { + "$ref": "#/definitions/StorageUsage" + }, + { + "type": "null" + } + ] + }, + "attached_instances": { + "type": [ + "array", + "null" + ], + "items": { + "$ref": "#/definitions/StorageAttachment" + } + } + } + } + }, + "additionalProperties": false + } + ] + }, + "InstanceUsage": { + "type": "object", + "required": [ + "average_cpu_load", + "state" + ], + "properties": { + "average_cpu_load": { + "type": "number", + "format": "double" + }, + "state": { + "$ref": "#/definitions/InstanceState" + } + } + }, + "InstanceState": { + "type": "string", + "enum": [ + "running", + "stopped" + ] + }, + "StorageUsage": { + "type": "object", + "required": [ + "size_gb" + ], + "properties": { + "size_gb": { + "type": "integer", + "format": "int32" + } + } + }, + "StorageAttachment": { + "type": "object", + "required": [ + "instance_id" + ], + "properties": { + "instance_id": { + "type": "string" + } + } + }, + "CloudResourceTag": { + "description": "A tag (just a mandatory key + optional value)", + "type": "object", + "required": [ + "key" + ], + "properties": { + "key": { + "type": "string" + }, + "value": { + "type": [ + "string", + "null" + ] + } + } + } + } +} \ No newline at end of file From 3e427b00ba242fc597f8d64192c6155a7ba3806b Mon Sep 17 00:00:00 2001 From: olivier de Meringo Date: Fri, 30 Aug 2024 11:56:48 +0200 Subject: [PATCH 2/2] doc: add command to generate JSON schema of inventory --- docs/src/how-to/simulate-impacts-of-an-inventory.md | 1 + docs/src/reference/cli-options.md | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/docs/src/how-to/simulate-impacts-of-an-inventory.md b/docs/src/how-to/simulate-impacts-of-an-inventory.md index 43f3c5d4..06e20760 100644 --- a/docs/src/how-to/simulate-impacts-of-an-inventory.md +++ b/docs/src/how-to/simulate-impacts-of-an-inventory.md @@ -9,3 +9,4 @@ This involves building an inventory file and passing it to cloud scanner for eva 💡 It may be easier to adapt an existing inventory file, rather than creating it from scratch. See [Estimate the impacts of an existing inventory](estimate-from-existing-inventory-file.md). +The JSON schema of the inventory file is in the git repository: [cloud-scanner/cloud-scanner-cli/test-data/INVENTORY_JSON_SCHEMA.json](https://github.com/Boavizta/cloud-scanner/blob/main/cloud-scanner-cli/test-data/INVENTORY_JSON_SCHEMA.json). This schema can also be retrieved with the command `cargo run inventory --print-json-schema`. diff --git a/docs/src/reference/cli-options.md b/docs/src/reference/cli-options.md index 51b09625..e8a5ce6b 100644 --- a/docs/src/reference/cli-options.md +++ b/docs/src/reference/cli-options.md @@ -38,6 +38,13 @@ Use the `--include-block-storage` command line flag or parameter to consider blo cargo run estimate --use-duration-hours 1 --include-block-storage --output-verbose-json ``` +## Print the JSON schema of the inventory file + +```sh +# Print the JSON schema of an inventory file (without producing the inventory) +cargo run inventory --print-json-schema +``` + ## Display statistics Use `-v` will display statistics on std error.