From 43fbbc1f1d95908444e990e1aa9e6c706b0b8fbe Mon Sep 17 00:00:00 2001 From: Alex Tonkonozhenko Date: Tue, 5 Mar 2024 17:48:07 +0100 Subject: [PATCH 1/2] Do not require enable_dynamic_partitioning for dynamic_partition_append_delimiter_to_record --- locals.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/locals.tf b/locals.tf index b051d2a..1b8788a 100644 --- a/locals.tf +++ b/locals.tf @@ -29,7 +29,7 @@ locals { is_search_destination = contains(["elasticsearch", "opensearch", "opensearchserverless"], local.destination) ? true : false # Data Transformation - enable_processing = var.enable_lambda_transform || var.enable_dynamic_partitioning || var.enable_cloudwatch_logs_decompression + enable_processing = var.enable_lambda_transform || var.enable_dynamic_partitioning || var.enable_cloudwatch_logs_decompression || var.dynamic_partition_append_delimiter_to_record lambda_processor = var.enable_lambda_transform ? { type = "Lambda" parameters = [ @@ -68,7 +68,7 @@ locals { }, ] } : null - append_delimiter_processor = var.enable_dynamic_partitioning && var.dynamic_partition_append_delimiter_to_record ? { + append_delimiter_processor = var.dynamic_partition_append_delimiter_to_record ? { type = "AppendDelimiterToRecord" parameters = [] } : null From a1d372e5f5d5df8d46785e82452e8ef1dedcb8f4 Mon Sep 17 00:00:00 2001 From: Fabio Santos Date: Tue, 5 Mar 2024 21:35:47 +0000 Subject: [PATCH 2/2] Deprecate dynamic_partition_append_delimiter_to_record variable --- README.md | 6 ++++++ examples/s3/kinesis-to-s3-complete/main.tf | 2 +- locals.tf | 4 ++-- variables.tf | 8 +++++++- 4 files changed, 16 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 3be1419..84d94ee 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,7 @@ Supports all destinations and all Kinesis Firehose Features. * [Upgrade](#upgrade) * [Deprecations](#deprecations) * [Version 3.1.0](#version-310) + * [Version 3.3.0](#version-330) * [License](#license) ## Module versioning rule @@ -912,6 +913,7 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| +| [append\_delimiter\_to\_record](#input\_append\_delimiter\_to\_record) | To configure your delivery stream to add a new line delimiter between records in objects that are delivered to Amazon S3. | `bool` | `false` | no | | [application\_role\_description](#input\_application\_role\_description) | Description of IAM Application role to use for Kinesis Firehose Stream Source | `string` | `null` | no | | [application\_role\_force\_detach\_policies](#input\_application\_role\_force\_detach\_policies) | Specifies to force detaching any policies the IAM Application role has before destroying it | `bool` | `true` | no | | [application\_role\_name](#input\_application\_role\_name) | Name of IAM Application role to use for Kinesis Firehose Stream Source | `string` | `null` | no | @@ -1140,6 +1142,10 @@ No modules. * Variable `kinesis_source_role_arn` is deprecated. Use `source_role_arn` instead. * Variable `kinesis_source_use_existing_role` is deprecated. Use `source_use_existing_role` instead. +### Version 3.3.0 + +* Variable `dynamic_partition_append_delimiter_to_record` is deprecated. Use `append_delimiter_to_record` instead. + ## License Apache 2 Licensed. See [LICENSE](https://github.com/fdmsantos/terraform-aws-kinesis-firehose/tree/main/LICENSE) for full details. diff --git a/examples/s3/kinesis-to-s3-complete/main.tf b/examples/s3/kinesis-to-s3-complete/main.tf index e037d51..3d870bc 100644 --- a/examples/s3/kinesis-to-s3-complete/main.tf +++ b/examples/s3/kinesis-to-s3-complete/main.tf @@ -73,6 +73,6 @@ module "firehose" { enable_dynamic_partitioning = true dynamic_partitioning_retry_duration = 350 dynamic_partition_metadata_extractor_query = "{user_id:.user_id}" - dynamic_partition_append_delimiter_to_record = true + append_delimiter_to_record = true dynamic_partition_enable_record_deaggregation = true } diff --git a/locals.tf b/locals.tf index 1b8788a..302ef29 100644 --- a/locals.tf +++ b/locals.tf @@ -29,7 +29,7 @@ locals { is_search_destination = contains(["elasticsearch", "opensearch", "opensearchserverless"], local.destination) ? true : false # Data Transformation - enable_processing = var.enable_lambda_transform || var.enable_dynamic_partitioning || var.enable_cloudwatch_logs_decompression || var.dynamic_partition_append_delimiter_to_record + enable_processing = var.enable_lambda_transform || var.enable_dynamic_partitioning || var.enable_cloudwatch_logs_decompression || var.dynamic_partition_append_delimiter_to_record || var.append_delimiter_to_record lambda_processor = var.enable_lambda_transform ? { type = "Lambda" parameters = [ @@ -68,7 +68,7 @@ locals { }, ] } : null - append_delimiter_processor = var.dynamic_partition_append_delimiter_to_record ? { + append_delimiter_processor = var.dynamic_partition_append_delimiter_to_record || var.append_delimiter_to_record ? { type = "AppendDelimiterToRecord" parameters = [] } : null diff --git a/variables.tf b/variables.tf index 36988f8..5f455b7 100644 --- a/variables.tf +++ b/variables.tf @@ -461,7 +461,7 @@ variable "dynamic_partitioning_retry_duration" { } } -variable "dynamic_partition_append_delimiter_to_record" { +variable "dynamic_partition_append_delimiter_to_record" { # TODO Variable Deprecated. Remove on Next Major Version description = "To configure your delivery stream to add a new line delimiter between records in objects that are delivered to Amazon S3." type = bool default = false @@ -721,6 +721,12 @@ variable "data_format_conversion_orc_stripe_size" { } } +variable "append_delimiter_to_record" { + description = "To configure your delivery stream to add a new line delimiter between records in objects that are delivered to Amazon S3." + type = bool + default = false +} + ###### # Redshift Destination Variables ######