Skip to content

Commit

Permalink
Merge pull request #516 from Boavizta/508-add-metadata-to-the-inventory
Browse files Browse the repository at this point in the history
Feat: add metadata to the JSON inventory (optional date and comment).
demeringo authored May 30, 2024
2 parents 64c31de + 78fe7bb commit 6faaac7
Showing 10 changed files with 386 additions and 104 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
_This paragraph may describe WIP/unreleased features. They are merged to main branch but not tagged._

- [352 estimate impacts of an existing inventory by demeringo · Pull Request #505 · Boavizta/cloud-scanner](https://github.com/Boavizta/cloud-scanner/pull/505). ⚠ This introduces breaking changes on the CLI options. The option to get results as metrics (using the flag `--as-metrics` on the 'estimate' command is replaced by a direct command name `metrics`).
- [Add metadata to the inventory · Issue #508 · Boavizta/cloud-scanner](https://github.com/Boavizta/cloud-scanner/issues/508)

## [2.0.5]-2024-04-12

171 changes: 106 additions & 65 deletions Cargo.lock

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion cloud-scanner-cli/Cargo.toml
Original file line number Diff line number Diff line change
@@ -5,7 +5,7 @@ name = "cloud-scanner-cli"
version = "2.0.5"

[dependencies]
chrono = "^0.4"
chrono = { version = "^0.4", features = ["serde"] }
isocountry = "^0.3"
log = "0.4"
loggerv = "0.7"
@@ -21,6 +21,7 @@ rocket = { version = "0.5.0", default-features = false, features = [
"json",
] }
rocket_okapi = { version = "0.8.0", features = ["swagger", "rapidoc"] }
schemars = { version = "0.8", features = ["chrono"] }
aws-types = "1"
thiserror = "1.0.57"

78 changes: 44 additions & 34 deletions cloud-scanner-cli/src/aws_cloud_provider.rs
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
//! A module to perform inventory of AWS cloud resources.
//! A module that returns inventory of AWS resources.
use std::time::Instant;

use crate::cloud_provider::Inventoriable;
@@ -8,19 +8,17 @@ use anyhow::{Context, Error, Result};
use aws_sdk_cloudwatch::operation::get_metric_statistics::GetMetricStatisticsOutput;
use aws_sdk_cloudwatch::types::{Dimension, StandardUnit, Statistic};
use aws_sdk_ec2::config::Region;
use aws_sdk_ec2::types::Volume;
use aws_sdk_ec2::types::{Instance, InstanceStateName};
use chrono::TimeDelta;
use chrono::Utc;
use aws_sdk_ec2::types::{Instance, InstanceStateName, Volume};
use chrono::{TimeDelta, Utc};

use crate::model::{
CloudProvider, CloudResource, CloudResourceTag, ExecutionStatistics, InstanceState,
InstanceUsage, Inventory, ResourceDetails, StorageAttachment, StorageUsage,
InstanceUsage, Inventory, InventoryMetadata, ResourceDetails, StorageAttachment, StorageUsage,
};
use async_trait::async_trait;
use aws_types::SdkConfig;

/// An service to perform inventory of AWS resources.
/// A service that returns inventory of AWS resources.
#[derive(Clone, Debug)]
pub struct AwsCloudProvider {
aws_region: String,
@@ -29,42 +27,48 @@ pub struct AwsCloudProvider {
}

impl AwsCloudProvider {
/// Creates a service to perform inventory of AWS resources.
/// Creates a service that returns inventory of AWS resources.
///
/// Initializes it with a specific region and configures the SDK's that will query your account to perform the inventory of resources.
pub async fn new(aws_region: &str) -> Self {
let shared_config = Self::load_aws_config(aws_region).await;
pub async fn new(aws_region: &str) -> Result<Self> {
let shared_config = Self::load_aws_config(aws_region).await?;
let retained_region = Self::get_configured_region_or_exit_if_unsupported(&shared_config);

AwsCloudProvider {
Ok(AwsCloudProvider {
aws_region: retained_region,
ec2_client: aws_sdk_ec2::Client::new(&shared_config),
cloudwatch_client: aws_sdk_cloudwatch::Client::new(&shared_config),
}
})
}

/// Initialize a AWS SDK config with default credentials from the environment and a region passed as argument.
/// Initialize an AWS SDK config using credentials from the environment and a region passed as argument.
///
/// - If region is empty, uses the default region from environment.
/// - ⚠ If the region is invalid, it does **not** return error.
async fn load_aws_config(aws_region: &str) -> SdkConfig {
async fn load_aws_config(aws_region: &str) -> Result<SdkConfig> {
if aws_region.is_empty() {
// Use default region (from environment, if any)
let sdk_config = aws_config::load_from_env().await;
warn!(
"Cannot initialize AWS client from an empty region, falling back to using default region from environment [{}]",
sdk_config.region().unwrap()
);

sdk_config
match sdk_config.region() {
None => {
warn!("Tried to initialize AWS client without a region.");
}
Some(region) => {
warn!(
"Initialized AWS client from the default region picked up from environment [{}]", region
);
}
}
Ok(sdk_config)
} else {
// Use the region passed in argument
let sdk_config = aws_config::from_env()
.region(Region::new(aws_region.to_string()))
.load()
.await;
info!("Initialized AWS client with with region [{}]", aws_region);
sdk_config
Ok(sdk_config)
}
}

@@ -106,8 +110,7 @@ impl AwsCloudProvider {
.clone()
.list_instances(tags)
.await
.context("Cannot list instances")
.unwrap();
.context("Cannot list instances")?;
let location = UsageLocation::try_from(self.aws_region.as_str())?;

// Just to display statistics
@@ -120,8 +123,7 @@ impl AwsCloudProvider {
.clone()
.get_average_cpu(&instance_id)
.await
.context("Cannot get CPU load of instance")
.unwrap();
.context("Cannot get CPU load of instance")?;

let usage: InstanceUsage = InstanceUsage {
average_cpu_load: cpuload,
@@ -365,7 +367,13 @@ impl Inventoriable for AwsCloudProvider {
};
warn!("{:?}", stats);

let metadata = InventoryMetadata {
inventory_date: Some(Utc::now()),
description: Some(String::from("About this inventory")),
};

let inventory = Inventory {
metadata,
resources,
execution_statistics: Some(stats),
};
@@ -383,7 +391,7 @@ mod tests {
#[tokio::test]
#[ignore]
async fn inventory_should_return_correct_number_of_instances() {
let aws: AwsCloudProvider = AwsCloudProvider::new("eu-west-1").await;
let aws: AwsCloudProvider = AwsCloudProvider::new("eu-west-1").await.unwrap();
let filtertags: Vec<String> = Vec::new();
let res: Vec<CloudResource> = aws
.get_instances_with_usage_data(&filtertags)
@@ -406,18 +414,20 @@ mod tests {
#[tokio::test]
async fn test_create_sdk_config_works_with_wrong_region() {
let region: &str = "eu-west-3";
let config = AwsCloudProvider::load_aws_config(region).await;
let config = AwsCloudProvider::load_aws_config(region).await.unwrap();
assert_eq!(region, config.region().unwrap().to_string());

let wrong_region: &str = "impossible-region";
let config = AwsCloudProvider::load_aws_config(wrong_region).await;
let config = AwsCloudProvider::load_aws_config(wrong_region)
.await
.unwrap();
assert_eq!(wrong_region, config.region().unwrap().to_string())
}

#[tokio::test]
#[ignore]
async fn get_cpu_usage_metrics_of_running_instance_should_return_right_number_of_data_points() {
let aws: AwsCloudProvider = AwsCloudProvider::new("eu-west-1").await;
let aws: AwsCloudProvider = AwsCloudProvider::new("eu-west-1").await.unwrap();
let res = aws
.get_average_cpu_usage_of_last_10_minutes(&RUNNING_INSTANCE_ID)
.await
@@ -435,7 +445,7 @@ mod tests {
#[tokio::test]
#[ignore]
async fn test_get_instance_usage_metrics_of_shutdown_instance() {
let aws: AwsCloudProvider = AwsCloudProvider::new("eu-west-1").await;
let aws: AwsCloudProvider = AwsCloudProvider::new("eu-west-1").await.unwrap();
let instance_id = "i-03e0b3b1246001382";
let res = aws
.get_average_cpu_usage_of_last_10_minutes(instance_id)
@@ -448,7 +458,7 @@ mod tests {
#[tokio::test]
#[ignore]
async fn test_get_instance_usage_metrics_of_non_existing_instance() {
let aws: AwsCloudProvider = AwsCloudProvider::new("eu-west-1").await;
let aws: AwsCloudProvider = AwsCloudProvider::new("eu-west-1").await.unwrap();
let instance_id = "IDONOTEXISTS";
let res = aws
.get_average_cpu_usage_of_last_10_minutes(instance_id)
@@ -462,7 +472,7 @@ mod tests {
#[ignore]
async fn test_average_cpu_load_of_running_instance_is_not_zero() {
// This instance needs to be running for the test to pass
let aws: AwsCloudProvider = AwsCloudProvider::new("eu-west-1").await;
let aws: AwsCloudProvider = AwsCloudProvider::new("eu-west-1").await.unwrap();

let avg_cpu_load = aws.get_average_cpu(&RUNNING_INSTANCE_ID).await.unwrap();
assert_ne!(
@@ -479,15 +489,15 @@ mod tests {
#[ignore]
async fn test_average_cpu_load_of_non_existing_instance_is_zero() {
let instance_id = "IDONOTEXISTS";
let aws: AwsCloudProvider = AwsCloudProvider::new("eu-west-1").await;
let aws: AwsCloudProvider = AwsCloudProvider::new("eu-west-1").await.unwrap();
let res = aws.get_average_cpu(instance_id).await.unwrap();
assert_eq!(0 as f64, res);
}

#[tokio::test]
#[ignore]
async fn test_average_cpu_load_of_shutdown_instance_is_zero() {
let aws: AwsCloudProvider = AwsCloudProvider::new("eu-west-1").await;
let aws: AwsCloudProvider = AwsCloudProvider::new("eu-west-1").await.unwrap();
let instance_id = "i-03e0b3b1246001382";
let res = aws.get_average_cpu(instance_id).await.unwrap();
assert_eq!(0 as f64, res);
@@ -496,7 +506,7 @@ mod tests {
#[tokio::test]
#[ignore]
async fn returns_the_right_number_of_volumes() {
let aws: AwsCloudProvider = AwsCloudProvider::new("eu-west-1").await;
let aws: AwsCloudProvider = AwsCloudProvider::new("eu-west-1").await.unwrap();
let filtertags: Vec<String> = Vec::new();
let res = aws.list_volumes(&filtertags).await.unwrap();
assert_eq!(4, res.len());
11 changes: 10 additions & 1 deletion cloud-scanner-cli/src/boavizta_api_v1.rs
Original file line number Diff line number Diff line change
@@ -294,7 +294,8 @@ mod tests {

use super::*;
use crate::model::{
CloudProvider, CloudResource, InstanceState, InstanceUsage, ResourceDetails, StorageUsage,
CloudProvider, CloudResource, InstanceState, InstanceUsage, InventoryMetadata,
ResourceDetails, StorageUsage,
};
use crate::usage_location::UsageLocation;
use assert_json_diff::assert_json_include;
@@ -447,6 +448,10 @@ mod tests {
instances.push(instance1_1percent);

let inventory = Inventory {
metadata: InventoryMetadata {
inventory_date: None,
description: None,
},
resources: instances,
execution_statistics: None,
};
@@ -510,6 +515,10 @@ mod tests {
let one_hour = 1.0 as f32;

let inventory = Inventory {
metadata: InventoryMetadata {
inventory_date: None,
description: None,
},
resources: instances,
execution_statistics: None,
};
4 changes: 2 additions & 2 deletions cloud-scanner-cli/src/lib.rs
Original file line number Diff line number Diff line change
@@ -45,7 +45,7 @@ pub async fn estimate_impacts(
verbose: bool,
include_block_storage: bool,
) -> Result<EstimatedInventory> {
let aws_provider: AwsCloudProvider = AwsCloudProvider::new(aws_region).await;
let aws_provider: AwsCloudProvider = AwsCloudProvider::new(aws_region).await?;
let inventory: Inventory = aws_provider
.list_resources(tags, include_block_storage)
.await
@@ -132,7 +132,7 @@ pub async fn get_inventory(
aws_region: &str,
include_block_storage: bool,
) -> Result<Inventory> {
let aws_inventory: AwsCloudProvider = AwsCloudProvider::new(aws_region).await;
let aws_inventory: AwsCloudProvider = AwsCloudProvider::new(aws_region).await?;
let inventory: Inventory = aws_inventory
.list_resources(tags, include_block_storage)
.await
21 changes: 21 additions & 0 deletions cloud-scanner-cli/src/model.rs
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
//! Business Entities of cloud Scanner
use anyhow::Context;
use chrono::{DateTime, Utc};
use rocket_okapi::okapi::schemars;
use rocket_okapi::okapi::schemars::JsonSchema;
use serde::{Deserialize, Serialize};
@@ -29,10 +30,19 @@ impl fmt::Display for ExecutionStatistics {
#[derive(Clone, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct Inventory {
pub metadata: InventoryMetadata,
pub resources: Vec<CloudResource>,
pub execution_statistics: Option<ExecutionStatistics>,
}

/// Details about the inventory
#[derive(Clone, Serialize, Deserialize, JsonSchema)]
#[serde(rename_all = "camelCase")]
pub struct InventoryMetadata {
pub inventory_date: Option<DateTime<Utc>>,
pub description: Option<String>,
}

/// Load inventory from a file
pub async fn load_inventory_from_file(inventory_file_path: &Path) -> anyhow::Result<Inventory> {
let content = fs::read_to_string(inventory_file_path).context("cannot read inventory file")?;
@@ -388,4 +398,15 @@ mod tests {
"Wrong number of resources in the inventory file"
);
}

#[tokio::test]
async fn test_load_inventory_from_formatted_file() {
let inventory_file_path: &Path = Path::new("./test-data/AWS_INVENTORY_FORMATTED.json");
let inventory: Inventory = load_inventory_from_file(inventory_file_path).await.unwrap();
assert_eq!(
inventory.resources.len(),
2,
"Wrong number of resources in the inventory file"
);
}
}
2 changes: 1 addition & 1 deletion cloud-scanner-cli/test-data/AWS_INVENTORY.json
Original file line number Diff line number Diff line change
@@ -1 +1 @@
{"resources":[{"provider":"AWS","id":"i-03c8f84a6318a8186","location":{"aws_region":"eu-west-1","iso_country_code":"IRL"},"resource_details":{"Instance":{"instance_type":"a1.medium","usage":{"average_cpu_load":0.24541666666666664,"usage_duration_seconds":300,"state":"Running"}}},"tags":[{"key":"Name","value":"test-boapi"},{"key":"CreatorName","value":"olivierdemeringoadm"},{"key":"CustomTagNameForDebug","value":"olivierdemeringoadm"}]},{"provider":"AWS","id":"i-033df52f12f30ca66","location":{"aws_region":"eu-west-1","iso_country_code":"IRL"},"resource_details":{"Instance":{"instance_type":"m6g.xlarge","usage":{"average_cpu_load":0.0,"usage_duration_seconds":300,"state":"Stopped"}}},"tags":[{"key":"Name","value":"test-boavizta"},{"key":"CustomTagNameForDebug","value":"olivierdemeringoadm"},{"key":"CreatorName","value":"olivierdemeringoadm"}]},{"provider":"AWS","id":"i-0a3e6b8cdb50c49b8","location":{"aws_region":"eu-west-1","iso_country_code":"IRL"},"resource_details":{"Instance":{"instance_type":"c5n.xlarge","usage":{"average_cpu_load":0.0,"usage_duration_seconds":300,"state":"Stopped"}}},"tags":[{"key":"CustomTagNameForDebug","value":"olivierdemeringoadm"},{"key":"appname","value":"app1"},{"key":"created_by","value":"demeringo"},{"key":"CreatorName","value":"olivierdemeringoadm"},{"key":"Name","value":"boavizta-c5n.xlarge"}]},{"provider":"AWS","id":"i-003ea8da7bb9bfff9","location":{"aws_region":"eu-west-1","iso_country_code":"IRL"},"resource_details":{"Instance":{"instance_type":"m6g.xlarge","usage":{"average_cpu_load":0.05362499999999998,"usage_duration_seconds":300,"state":"Running"}}},"tags":[{"key":"CustomTagNameForDebug","value":"olivierdemeringoadm"},{"key":"CreatorName","value":"olivierdemeringoadm"},{"key":"Name","value":"test-boavizta-2"}]}],"executionStatistics":{"inventory_duration":{"secs":0,"nanos":669819838},"impact_estimation_duration":{"secs":0,"nanos":0},"total_duration":{"secs":0,"nanos":669820304}}}
{"metadata":{},"resources":[{"provider":"AWS","id":"i-03c8f84a6318a8186","location":{"aws_region":"eu-west-1","iso_country_code":"IRL"},"resource_details":{"Instance":{"instance_type":"a1.medium","usage":{"average_cpu_load":0.24541666666666664,"state":"Running"}}},"tags":[{"key":"Name","value":"test-boapi"},{"key":"CreatorName","value":"olivierdemeringoadm"},{"key":"CustomTagNameForDebug","value":"olivierdemeringoadm"}]},{"provider":"AWS","id":"i-033df52f12f30ca66","location":{"aws_region":"eu-west-1","iso_country_code":"IRL"},"resource_details":{"Instance":{"instance_type":"m6g.xlarge","usage":{"average_cpu_load":0.0,"state":"Stopped"}}},"tags":[{"key":"Name","value":"test-boavizta"},{"key":"CustomTagNameForDebug","value":"olivierdemeringoadm"},{"key":"CreatorName","value":"olivierdemeringoadm"}]},{"provider":"AWS","id":"i-0a3e6b8cdb50c49b8","location":{"aws_region":"eu-west-1","iso_country_code":"IRL"},"resource_details":{"Instance":{"instance_type":"c5n.xlarge","usage":{"average_cpu_load":0.0,"state":"Stopped"}}},"tags":[{"key":"CustomTagNameForDebug","value":"olivierdemeringoadm"},{"key":"appname","value":"app1"},{"key":"created_by","value":"demeringo"},{"key":"CreatorName","value":"olivierdemeringoadm"},{"key":"Name","value":"boavizta-c5n.xlarge"}]},{"provider":"AWS","id":"i-003ea8da7bb9bfff9","location":{"aws_region":"eu-west-1","iso_country_code":"IRL"},"resource_details":{"Instance":{"instance_type":"m6g.xlarge","usage":{"average_cpu_load":0.05362499999999998,"state":"Running"}}},"tags":[{"key":"CustomTagNameForDebug","value":"olivierdemeringoadm"},{"key":"CreatorName","value":"olivierdemeringoadm"},{"key":"Name","value":"test-boavizta-2"}]}],"executionStatistics":{"inventory_duration":{"secs":0,"nanos":669819838},"impact_estimation_duration":{"secs":0,"nanos":0},"total_duration":{"secs":0,"nanos":669820304}}}
44 changes: 44 additions & 0 deletions cloud-scanner-cli/test-data/AWS_INVENTORY_FORMATTED.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,44 @@
{
"metadata": {
"inventoryDate": "2024-05-30 12:21:03.0Z",
"description": "A test inventory"
},
"resources": [
{
"provider": "AWS",
"id": "instance-1",
"location": {
"aws_region": "eu-west-1",
"iso_country_code": "IRL"
},
"resource_details": {
"Instance": {
"instance_type": "a1.medium",
"usage": {
"average_cpu_load": 0.3,
"state": "Running"
}
}
},
"tags": []
},
{
"provider": "AWS",
"id": "instance-2",
"location": {
"aws_region": "eu-west-1",
"iso_country_code": "IRL"
},
"resource_details": {
"Instance": {
"instance_type": "m6g.xlarge",
"usage": {
"average_cpu_load": 0,
"state": "Stopped"
}
}
},
"tags": []
}
]
}
155 changes: 155 additions & 0 deletions res.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,155 @@
{
"metadata": {},
"resources": [
{
"provider": "AWS",
"id": "i-03c8f84a6318a8186",
"location": {
"aws_region": "eu-west-1",
"iso_country_code": "IRL"
},
"resource_details": {
"Instance": {
"instance_type": "a1.medium",
"usage": {
"average_cpu_load": 0.24541666666666664,
"usage_duration_seconds": 300,
"state": "Running"
}
}
},
"tags": [
{
"key": "Name",
"value": "test-boapi"
},
{
"key": "CreatorName",
"value": "olivierdemeringoadm"
},
{
"key": "CustomTagNameForDebug",
"value": "olivierdemeringoadm"
}
]
},
{
"provider": "AWS",
"id": "i-033df52f12f30ca66",
"location": {
"aws_region": "eu-west-1",
"iso_country_code": "IRL"
},
"resource_details": {
"Instance": {
"instance_type": "m6g.xlarge",
"usage": {
"average_cpu_load": 0,
"usage_duration_seconds": 300,
"state": "Stopped"
}
}
},
"tags": [
{
"key": "Name",
"value": "test-boavizta"
},
{
"key": "CustomTagNameForDebug",
"value": "olivierdemeringoadm"
},
{
"key": "CreatorName",
"value": "olivierdemeringoadm"
}
]
},
{
"provider": "AWS",
"id": "i-0a3e6b8cdb50c49b8",
"location": {
"aws_region": "eu-west-1",
"iso_country_code": "IRL"
},
"resource_details": {
"Instance": {
"instance_type": "c5n.xlarge",
"usage": {
"average_cpu_load": 0,
"usage_duration_seconds": 300,
"state": "Stopped"
}
}
},
"tags": [
{
"key": "CustomTagNameForDebug",
"value": "olivierdemeringoadm"
},
{
"key": "appname",
"value": "app1"
},
{
"key": "created_by",
"value": "demeringo"
},
{
"key": "CreatorName",
"value": "olivierdemeringoadm"
},
{
"key": "Name",
"value": "boavizta-c5n.xlarge"
}
]
},
{
"provider": "AWS",
"id": "i-003ea8da7bb9bfff9",
"location": {
"aws_region": "eu-west-1",
"iso_country_code": "IRL"
},
"resource_details": {
"Instance": {
"instance_type": "m6g.xlarge",
"usage": {
"average_cpu_load": 0.05362499999999998,
"usage_duration_seconds": 300,
"state": "Running"
}
}
},
"tags": [
{
"key": "CustomTagNameForDebug",
"value": "olivierdemeringoadm"
},
{
"key": "CreatorName",
"value": "olivierdemeringoadm"
},
{
"key": "Name",
"value": "test-boavizta-2"
}
]
}
],
"executionStatistics": {
"inventory_duration": {
"secs": 0,
"nanos": 669819838
},
"impact_estimation_duration": {
"secs": 0,
"nanos": 0
},
"total_duration": {
"secs": 0,
"nanos": 669820304
}
}
}

0 comments on commit 6faaac7

Please sign in to comment.