diff --git a/Cargo.lock b/Cargo.lock index 25d73b3ab20f2..53716f4061b2b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -84,9 +84,9 @@ dependencies = [ [[package]] name = "ahash" -version = "0.7.7" +version = "0.7.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a824f2aa7e75a0c98c5a504fceb80649e9c35265d44525b5f94de4771a395cd" +checksum = "891477e0c6a8957309ee5c45a6368af3ae14bb510732d2684ffa19af310920f9" dependencies = [ "getrandom 0.2.15", "once_cell", @@ -788,7 +788,7 @@ dependencies = [ "aws-sdk-sts", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.60.7", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -835,22 +835,26 @@ dependencies = [ [[package]] name = "aws-runtime" -version = "1.0.3" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ce0953f7fc1c4428511345e28ea3e98c8b59c9e91eafae30bf76d71d70642693" +checksum = "300a12520b4e6d08b73f77680f12c16e8ae43250d55100e0b2be46d78da16a48" dependencies = [ "aws-credential-types", - "aws-http", "aws-sigv4", "aws-smithy-async", "aws-smithy-eventstream", "aws-smithy-http", + "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", "aws-types", + "bytes 1.9.0", "fastrand 2.1.1", "http 0.2.9", + "http-body 0.4.5", + "once_cell", "percent-encoding", + "pin-project-lite", "tracing 0.1.41", "uuid", ] @@ -866,7 +870,7 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.60.7", "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -880,16 +884,15 @@ dependencies = [ [[package]] name = "aws-sdk-cloudwatchlogs" -version = "1.3.0" +version = "1.62.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67a5bcf460e098cf49292d216fe520b28a5d9c8dae10db0ff9a97bb2c95dd386" +checksum = "2cbddf4d7c93d80b67ca47af16e37720fde7c62e8e69b371ff0977bd9d94ade1" dependencies = [ "aws-credential-types", - "aws-http", "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.61.1", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -897,7 +900,8 @@ dependencies = [ "bytes 1.9.0", "fastrand 2.1.1", "http 0.2.9", - "regex", + "once_cell", + "regex-lite", "tracing 0.1.41", ] @@ -912,7 +916,7 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.60.7", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -934,7 +938,7 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.60.7", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -956,7 +960,7 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.60.7", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -981,7 +985,7 @@ dependencies = [ "aws-smithy-checksums", "aws-smithy-eventstream", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.60.7", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -1008,7 +1012,7 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.60.7", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -1031,7 +1035,7 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.60.7", "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -1054,7 +1058,7 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.60.7", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -1076,7 +1080,7 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.60.7", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -1098,7 +1102,7 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.60.7", "aws-smithy-runtime", "aws-smithy-runtime-api", "aws-smithy-types", @@ -1120,7 +1124,7 @@ dependencies = [ "aws-runtime", "aws-smithy-async", "aws-smithy-http", - "aws-smithy-json", + "aws-smithy-json 0.60.7", "aws-smithy-query", "aws-smithy-runtime", "aws-smithy-runtime-api", @@ -1134,9 +1138,9 @@ dependencies = [ [[package]] name = "aws-sigv4" -version = "1.2.5" +version = "1.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5619742a0d8f253be760bfbb8e8e8368c69e3587e4637af5754e488a611499b1" +checksum = "7d3820e0c08d0737872ff3c7c1f21ebbb6693d832312d6152bf18ef50a5471c2" dependencies = [ "aws-credential-types", "aws-smithy-eventstream", @@ -1158,9 +1162,9 @@ dependencies = [ [[package]] name = "aws-smithy-async" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62220bc6e97f946ddd51b5f1361f78996e704677afc518a4ff66b7a72ea1378c" +checksum = "8aa8ff1492fd9fb99ae28e8467af0dbbb7c31512b16fabf1a0f10d7bb6ef78bb" dependencies = [ "futures-util", "pin-project-lite", @@ -1229,6 +1233,15 @@ dependencies = [ "aws-smithy-types", ] +[[package]] +name = "aws-smithy-json" +version = "0.61.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee4e69cc50921eb913c6b662f8d909131bb3e6ad6cb6090d3a39b66fc5c52095" +dependencies = [ + "aws-smithy-types", +] + [[package]] name = "aws-smithy-query" version = "0.60.7" @@ -1241,9 +1254,9 @@ dependencies = [ [[package]] name = "aws-smithy-runtime" -version = "1.7.4" +version = "1.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f20685047ca9d6f17b994a07f629c813f08b5bce65523e47124879e60103d45" +checksum = "431a10d0e07e09091284ef04453dae4069283aa108d209974d67e77ae1caa658" dependencies = [ "aws-smithy-async", "aws-smithy-http", @@ -1285,9 +1298,9 @@ dependencies = [ [[package]] name = "aws-smithy-types" -version = "1.2.9" +version = "1.2.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fbd94a32b3a7d55d3806fe27d98d3ad393050439dd05eb53ece36ec5e3d3510" +checksum = "8ecbf4d5dfb169812e2b240a4350f15ad3c6b03a54074e5712818801615f2dc5" dependencies = [ "base64-simd", "bytes 1.9.0", @@ -2500,9 +2513,9 @@ checksum = "19d374276b40fb8bbdee95aef7c7fa6b5316ec764510eb64b8dd0e2ed0d7e7f5" [[package]] name = "crc32c" -version = "0.6.4" +version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d8f48d60e5b4d2c53d5c2b1d8a58c849a70ae5e5509b08a48d047e3b65714a74" +checksum = "3a47af21622d091a8f0fb295b88bc886ac74efcc613efc19f5d0b21de5c89e47" dependencies = [ "rustc_version 0.4.1", ] @@ -4077,7 +4090,7 @@ version = "0.12.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888" dependencies = [ - "ahash 0.7.7", + "ahash 0.7.8", ] [[package]] @@ -6378,7 +6391,7 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56" dependencies = [ - "proc-macro-crate 1.3.1", + "proc-macro-crate 3.2.0", "proc-macro2 1.0.92", "quote 1.0.37", "syn 2.0.90", diff --git a/Cargo.toml b/Cargo.toml index d693da56644e9..0b7b22b59dd9e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -214,7 +214,7 @@ aws-sdk-s3 = { version = "1.4.0", default-features = false, features = ["behavio aws-sdk-sqs = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } aws-sdk-sns = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } aws-sdk-cloudwatch = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } -aws-sdk-cloudwatchlogs = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } +aws-sdk-cloudwatchlogs = { version = "1.62.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } aws-sdk-elasticsearch = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } aws-sdk-firehose = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } aws-sdk-kinesis = { version = "1.3.0", default-features = false, features = ["behavior-version-latest", "rt-tokio"], optional = true } diff --git a/changelog.d/22008-specify-cloudwatch-log-class.enhancement.md b/changelog.d/22008-specify-cloudwatch-log-class.enhancement.md new file mode 100644 index 0000000000000..af2c7789c2524 --- /dev/null +++ b/changelog.d/22008-specify-cloudwatch-log-class.enhancement.md @@ -0,0 +1,3 @@ +The Cloudwatch Logs Sink now supports specifying the type of log class to create. + +authors: PriceHiller diff --git a/src/sinks/aws_cloudwatch_logs/config.rs b/src/sinks/aws_cloudwatch_logs/config.rs index d84076884a37e..a115f0f0a7ded 100644 --- a/src/sinks/aws_cloudwatch_logs/config.rs +++ b/src/sinks/aws_cloudwatch_logs/config.rs @@ -76,6 +76,30 @@ where } } +/// Defines the log class to create +/// +/// See https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/CloudWatch_Logs_Log_Classes.html +#[configurable_component] +#[derive(Clone, Debug, Default)] +pub enum LogGroupClassDef { + /// Logs that require real-time monitoring or frequently accessed logs + #[default] + Standard, + /// Log class that can be used to cost-effectively consolidate logs + InfrequentAccess, +} + +impl From for aws_sdk_cloudwatchlogs::types::LogGroupClass { + fn from(value: LogGroupClassDef) -> Self { + match value { + LogGroupClassDef::Standard => aws_sdk_cloudwatchlogs::types::LogGroupClass::Standard, + LogGroupClassDef::InfrequentAccess => { + aws_sdk_cloudwatchlogs::types::LogGroupClass::InfrequentAccess + } + } + } +} + /// Configuration for the `aws_cloudwatch_logs` sink. #[configurable_component(sink( "aws_cloudwatch_logs", @@ -110,6 +134,7 @@ pub struct CloudwatchLogsSinkConfig { pub region: RegionOrEndpoint, /// Dynamically create a [log group][log_group] if it does not already exist. + /// This will create the log group with the group class specified by the `group_class` option. /// /// This ignores `create_missing_stream` directly after creating the group and creates /// the first stream. @@ -118,6 +143,14 @@ pub struct CloudwatchLogsSinkConfig { #[serde(default = "crate::serde::default_true")] pub create_missing_group: bool, + /// Specifies the specific [group class][group_class] to create when + /// `create_missing_group` is enabled. + /// + /// [group_class]: https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/CloudWatch_Logs_Log_Classes.html + #[configurable(derived)] + #[serde(default)] + pub group_class: LogGroupClassDef, + /// Dynamically create a [log stream][log_stream] if it does not already exist. /// /// [log_stream]: https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/Working-with-log-groups-and-streams.html @@ -236,6 +269,7 @@ fn default_config(encoding: EncodingConfig) -> CloudwatchLogsSinkConfig { CloudwatchLogsSinkConfig { encoding, group_name: Default::default(), + group_class: Default::default(), stream_name: Default::default(), region: Default::default(), create_missing_group: true, diff --git a/src/sinks/aws_cloudwatch_logs/integration_tests.rs b/src/sinks/aws_cloudwatch_logs/integration_tests.rs index 9eebc9dbbf5cc..ee4a30a023108 100644 --- a/src/sinks/aws_cloudwatch_logs/integration_tests.rs +++ b/src/sinks/aws_cloudwatch_logs/integration_tests.rs @@ -39,6 +39,7 @@ async fn cloudwatch_insert_log_event() { let config = CloudwatchLogsSinkConfig { stream_name: Template::try_from(stream_name.as_str()).unwrap(), group_name: Template::try_from(GROUP_NAME).unwrap(), + group_class: Default::default(), region: RegionOrEndpoint::with_both("us-east-1", cloudwatch_address().as_str()), encoding: TextSerializerConfig::default().into(), create_missing_group: true, @@ -90,6 +91,7 @@ async fn cloudwatch_insert_log_events_sorted() { let config = CloudwatchLogsSinkConfig { stream_name: Template::try_from(stream_name.as_str()).unwrap(), group_name: Template::try_from(GROUP_NAME).unwrap(), + group_class: Default::default(), region: RegionOrEndpoint::with_both("us-east-1", cloudwatch_address().as_str()), encoding: TextSerializerConfig::default().into(), create_missing_group: true, @@ -166,6 +168,7 @@ async fn cloudwatch_insert_out_of_range_timestamp() { let config = CloudwatchLogsSinkConfig { stream_name: Template::try_from(stream_name.as_str()).unwrap(), group_name: Template::try_from(GROUP_NAME).unwrap(), + group_class: Default::default(), region: RegionOrEndpoint::with_both("us-east-1", cloudwatch_address().as_str()), encoding: TextSerializerConfig::default().into(), create_missing_group: true, @@ -243,6 +246,7 @@ async fn cloudwatch_dynamic_group_and_stream_creation() { let config = CloudwatchLogsSinkConfig { stream_name: Template::try_from(stream_name.as_str()).unwrap(), group_name: Template::try_from(group_name.as_str()).unwrap(), + group_class: Default::default(), region: RegionOrEndpoint::with_both("us-east-1", cloudwatch_address().as_str()), encoding: TextSerializerConfig::default().into(), create_missing_group: true, @@ -299,6 +303,7 @@ async fn cloudwatch_insert_log_event_batched() { let config = CloudwatchLogsSinkConfig { stream_name: Template::try_from(stream_name.as_str()).unwrap(), group_name: Template::try_from(group_name.as_str()).unwrap(), + group_class: Default::default(), region: RegionOrEndpoint::with_both("us-east-1", cloudwatch_address().as_str()), encoding: TextSerializerConfig::default().into(), create_missing_group: true, @@ -350,6 +355,7 @@ async fn cloudwatch_insert_log_event_partitioned() { let config = CloudwatchLogsSinkConfig { group_name: Template::try_from(GROUP_NAME).unwrap(), stream_name: Template::try_from(format!("{}-{{{{key}}}}", stream_name)).unwrap(), + group_class: Default::default(), region: RegionOrEndpoint::with_both("us-east-1", cloudwatch_address().as_str()), encoding: TextSerializerConfig::default().into(), create_missing_group: true, @@ -443,6 +449,7 @@ async fn cloudwatch_healthcheck() { let config = CloudwatchLogsSinkConfig { stream_name: Template::try_from("test-stream").unwrap(), group_name: Template::try_from(GROUP_NAME).unwrap(), + group_class: Default::default(), region: RegionOrEndpoint::with_both("us-east-1", cloudwatch_address().as_str()), encoding: TextSerializerConfig::default().into(), create_missing_group: true, diff --git a/src/sinks/aws_cloudwatch_logs/request.rs b/src/sinks/aws_cloudwatch_logs/request.rs index 60d9e88a6bb05..67ea130715153 100644 --- a/src/sinks/aws_cloudwatch_logs/request.rs +++ b/src/sinks/aws_cloudwatch_logs/request.rs @@ -12,7 +12,7 @@ use aws_sdk_cloudwatchlogs::{ put_log_events::{PutLogEventsError, PutLogEventsOutput}, put_retention_policy::PutRetentionPolicyError, }, - types::InputLogEvent, + types::{InputLogEvent, LogGroupClass}, Client as CloudwatchLogsClient, }; use aws_smithy_runtime_api::client::{orchestrator::HttpResponse, result::SdkError}; @@ -38,6 +38,7 @@ struct Client { client: CloudwatchLogsClient, stream_name: String, group_name: String, + group_class: LogGroupClass, headers: IndexMap, retention_days: u32, } @@ -60,6 +61,7 @@ impl CloudwatchFuture { headers: IndexMap, stream_name: String, group_name: String, + group_class: LogGroupClass, create_missing_group: bool, create_missing_stream: bool, retention: Retention, @@ -72,6 +74,7 @@ impl CloudwatchFuture { client, stream_name, group_name, + group_class, headers, retention_days, }; @@ -288,10 +291,12 @@ impl Client { pub fn create_log_group(&self) -> ClientResult<(), CreateLogGroupError> { let client = self.client.clone(); let group_name = self.group_name.clone(); + let group_class = self.group_class.clone(); Box::pin(async move { client .create_log_group() .log_group_name(group_name) + .log_group_class(group_class) .send() .await?; Ok(()) diff --git a/src/sinks/aws_cloudwatch_logs/service.rs b/src/sinks/aws_cloudwatch_logs/service.rs index ab8ea09daf551..f578199441e9d 100644 --- a/src/sinks/aws_cloudwatch_logs/service.rs +++ b/src/sinks/aws_cloudwatch_logs/service.rs @@ -10,7 +10,7 @@ use aws_sdk_cloudwatchlogs::{ describe_log_streams::DescribeLogStreamsError, put_log_events::PutLogEventsError, put_retention_policy::PutRetentionPolicyError, }, - types::InputLogEvent, + types::{InputLogEvent, LogGroupClass}, Client as CloudwatchLogsClient, }; use aws_smithy_runtime_api::client::{orchestrator::HttpResponse, result::SdkError}; @@ -240,11 +240,14 @@ impl CloudwatchLogsSvc { let retention = config.retention.clone(); + let group_class = config.group_class.into(); + CloudwatchLogsSvc { headers, client, stream_name, group_name, + group_class, create_missing_group, create_missing_stream, retention, @@ -319,6 +322,7 @@ impl Service> for CloudwatchLogsSvc { self.headers.clone(), self.stream_name.clone(), self.group_name.clone(), + self.group_class.clone(), self.create_missing_group, self.create_missing_stream, self.retention.clone(), @@ -337,6 +341,7 @@ pub struct CloudwatchLogsSvc { headers: IndexMap, stream_name: String, group_name: String, + group_class: LogGroupClass, create_missing_group: bool, create_missing_stream: bool, retention: Retention, diff --git a/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue b/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue index 9e9aa670e2eb2..9455b9bc1fed4 100644 --- a/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue +++ b/website/cue/reference/components/sinks/base/aws_cloudwatch_logs.cue @@ -201,6 +201,7 @@ base: components: sinks: aws_cloudwatch_logs: configuration: { create_missing_group: { description: """ Dynamically create a [log group][log_group] if it does not already exist. + This will create the log group with the group class specified by the `group_class` option. This ignores `create_missing_stream` directly after creating the group and creates the first stream. @@ -566,6 +567,22 @@ base: components: sinks: aws_cloudwatch_logs: configuration: { required: false type: string: examples: ["http://127.0.0.0:5000/path/to/service"] } + group_class: { + description: """ + Specifies the specific [group class][group_class] to create when + `create_missing_group` is enabled. + + [group_class]: https://docs.aws.amazon.com/AmazonCloudWatch/latest/logs/CloudWatch_Logs_Log_Classes.html + """ + required: false + type: string: { + default: "Standard" + enum: { + InfrequentAccess: "Log class that can be used to cost-effectively consolidate logs" + Standard: "Logs that require real-time monitoring or frequently accessed logs" + } + } + } group_name: { description: """ The [group name][group_name] of the target CloudWatch Logs stream.