From 7aa8e715fc8f56312a88214a8b78dd29869ee7bf Mon Sep 17 00:00:00 2001 From: Fabio Santos Date: Fri, 16 Feb 2024 23:56:18 +0000 Subject: [PATCH] BREAKING CHANGE: Add support for Opensearch and Opensearch Serverless destinations (#8) --- .github/workflows/pre-commit.yml | 2 +- CHANGELOG.md | 2 - README.md | 205 +++++++++----- UPGRADE-3.0.md | 36 +++ .../elasticsearch/opensearch-in-vpc/main.tf | 109 -------- .../opensearch-in-vpc/outputs.tf | 34 --- .../elasticsearch/public-opensearch/main.tf | 85 ------ .../public-opensearch/variables.tf | 19 -- .../direct-put-to-opensearch}/README.md | 8 +- .../direct-put-to-opensearch/main.tf | 83 ++++++ .../direct-put-to-opensearch}/outputs.tf | 0 .../direct-put-to-opensearch/variables.tf | 5 + .../direct-put-to-opensearch}/versions.tf | 0 .../README.md | 27 +- .../main.tf | 163 +++++++++++ .../outputs.tf | 4 + .../variables.tf | 16 +- .../versions.tf | 0 .../README.md | 64 +++++ .../main.tf | 124 +++++++++ .../outputs.tf | 4 + .../variables.tf | 5 + .../versions.tf | 14 + iam.tf | 188 +++++++++++-- locals.tf | 44 +-- main.tf | 255 +++++++++++++----- outputs.tf | 26 +- variables.tf | 121 +++++++-- versions.tf | 3 +- 29 files changed, 1152 insertions(+), 494 deletions(-) create mode 100644 UPGRADE-3.0.md delete mode 100644 examples/elasticsearch/opensearch-in-vpc/main.tf delete mode 100644 examples/elasticsearch/opensearch-in-vpc/outputs.tf delete mode 100644 examples/elasticsearch/public-opensearch/main.tf delete mode 100644 examples/elasticsearch/public-opensearch/variables.tf rename examples/{elasticsearch/public-opensearch => opensearch/direct-put-to-opensearch}/README.md (81%) create mode 100644 examples/opensearch/direct-put-to-opensearch/main.tf rename examples/{elasticsearch/public-opensearch => opensearch/direct-put-to-opensearch}/outputs.tf (100%) create mode 100644 examples/opensearch/direct-put-to-opensearch/variables.tf rename examples/{elasticsearch/opensearch-in-vpc => opensearch/direct-put-to-opensearch}/versions.tf (100%) rename examples/{elasticsearch/opensearch-in-vpc => opensearch/direct-put-to-opensearchserverless-in-vpc}/README.md (59%) create mode 100644 examples/opensearch/direct-put-to-opensearchserverless-in-vpc/main.tf create mode 100644 examples/opensearch/direct-put-to-opensearchserverless-in-vpc/outputs.tf rename examples/{elasticsearch/opensearch-in-vpc => opensearch/direct-put-to-opensearchserverless-in-vpc}/variables.tf (70%) rename examples/{elasticsearch/public-opensearch => opensearch/direct-put-to-opensearchserverless-in-vpc}/versions.tf (100%) create mode 100644 examples/opensearch/direct-put-to-opensearchserverless/README.md create mode 100644 examples/opensearch/direct-put-to-opensearchserverless/main.tf create mode 100644 examples/opensearch/direct-put-to-opensearchserverless/outputs.tf create mode 100644 examples/opensearch/direct-put-to-opensearchserverless/variables.tf create mode 100644 examples/opensearch/direct-put-to-opensearchserverless/versions.tf diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 7661604..cb82671 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -80,4 +80,4 @@ jobs: terraform-version: ${{ steps.minMax.outputs.maxVersion }} tflint-version: ${{ env.TFLINT_VERSION }} terraform-docs-version: ${{ env.TERRAFORM_DOCS_VERSION }} - install-hcledit: true \ No newline at end of file + install-hcledit: true diff --git a/CHANGELOG.md b/CHANGELOG.md index ff4e456..c51e31c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,13 +56,11 @@ All notable changes to this project will be documented in this file. ### ⚠ BREAKING CHANGES -* Bump Terraform AWS Provider version to 5.0 * Bump Terraform AWS Provider version to 5.0 ### Features * Bump Terraform AWS Provider version to 5.0 ([29601cf](https://github.com/fdmsantos/terraform-aws-kinesis-firehose/commit/29601cf8ad93a9a56ac3fba37f9e8b0c968c5c3f)) -* Bump Terraform AWS Provider version to 5.0 ([4be517c](https://github.com/fdmsantos/terraform-aws-kinesis-firehose/commit/4be517c9f13b62816cc3d0ccede495ab3715d009)) ### [1.9.1](https://github.com/fdmsantos/terraform-aws-kinesis-firehose/compare/v1.9.0...v1.9.1) (2023-09-16) diff --git a/README.md b/README.md index 2eca02c..55b2210 100644 --- a/README.md +++ b/README.md @@ -7,6 +7,7 @@ Supports all destinations and all Kinesis Firehose Features. ## Table of Contents +* [Table of Contents](#table-of-contents) * [Module versioning rule](#module-versioning-rule) * [Features](#features) * [How to Use](#how-to-use) @@ -14,11 +15,13 @@ Supports all destinations and all Kinesis Firehose Features. * [Kinesis Data Stream](#kinesis-data-stream) * [Kinesis Data Stream Encrypted](#kinesis-data-stream-encrypted) * [Direct Put](#direct-put) - * [WAF](waf) + * [WAF](#waf) * [Destinations](#destinations) * [S3](#s3) * [Redshift](#redshift) - * [Elasticsearch / Opensearch](#elasticsearch--opensearch) + * [Elasticsearch](#elasticsearch) + * [Opensearch](#opensearch) + * [Opensearch Serverless](#opensearch-serverless) * [Splunk](#splunk) * [HTTP Endpoint](#http-endpoint) * [Datadog](#datadog) @@ -36,7 +39,7 @@ Supports all destinations and all Kinesis Firehose Features. * [S3 Backup Data](#s3-backup-data) * [Destination Delivery Logging](#destination-delivery-logging) * [VPC Support](#vpc-support) - * [ElasticSearch / Opensearch](#elasticsearch--opensearch) + * [ElasticSearch / Opensearch / Opensearch Serverless](#elasticsearch--opensearch--opensearch-serverless) * [Redshift / Splunk](#redshift--splunk) * [Application Role](#application-role) * [Destinations Mapping](#destinations-mapping) @@ -48,15 +51,18 @@ Supports all destinations and all Kinesis Firehose Features. * [Inputs](#inputs) * [Outputs](#outputs) * [Deprecation](#deprecation) + * [Version >= 2.1.0](#version--210) * [Upgrade](#upgrade) * [License](#license) + ## Module versioning rule | Module version | AWS Provider version | |----------------|----------------------| | >= 1.x.x | ~> 4.4 | | >= 2.x.x | ~> 5.0 | +| >= 3.x.x | >= 5.33 | ## Features @@ -69,11 +75,11 @@ Supports all destinations and all Kinesis Firehose Features. - Data Format Conversion - Dynamic Partition - Redshift - - Redshift in VPC: Create Security Groups - - ElasticSearch / Opensearch - - VPC Support + - VPC Support. Security Groups creation supported + - ElasticSearch / Opensearch / Opensearch Serverless + - VPC Support. Security Groups creation supported - Splunk - - Splunk in VPC: Create Security Groups + - VPC Support. Security Groups creation supported - Custom Http Endpoint - DataDog - Coralogix @@ -89,10 +95,10 @@ Supports all destinations and all Kinesis Firehose Features. - Application Role to Direct Put Sources - Permissions - IAM Roles - - Opensearch Service Role + - Opensearch / Opensearch Serverless Service Role - Associate Role to Redshift Cluster Iam Roles - Cross Account S3 Bucket Policy - - Cross Account OpenSearch Service policy + - Cross Account Elasticsearch / OpenSearch / Opensearch Serverless Service policy ## How to Use @@ -194,9 +200,9 @@ module "firehose" { } ``` -#### Elasticsearch / Opensearch +#### Elasticsearch -**To Enabled It:** `destination = "elasticsearch" or destination = "opensearch"` +**To Enabled It:** `destination = "elasticsearch"` **Variables Prefix:** `elasticsearch_` @@ -205,12 +211,47 @@ module "firehose" { source = "fdmsantos/kinesis-firehose/aws" version = "x.x.x" name = "firehose-delivery-stream" - destination = "opensearch" # or destination = "elasticsearch" + destination = "elasticsearch" elasticsearch_domain_arn = "" elasticsearch_index_name = " [terraform](#requirement\_terraform) | >= 0.13.1 | -| [aws](#requirement\_aws) | ~> 5.0 | +| [aws](#requirement\_aws) | >= 5.33 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | ~> 5.0 | +| [aws](#provider\_aws) | >= 5.33 | ## Modules @@ -790,6 +834,8 @@ No modules. | [aws_iam_policy.glue](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_iam_policy.kinesis](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_iam_policy.lambda](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_policy.opensearch](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | +| [aws_iam_policy.opensearchserverless](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_iam_policy.s3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_iam_policy.s3_kms](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_iam_policy.vpc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | @@ -801,33 +847,40 @@ No modules. | [aws_iam_role_policy_attachment.glue](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.kinesis](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.lambda](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.opensearch](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_iam_role_policy_attachment.opensearchserverless](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.s3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.s3_kms](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_role_policy_attachment.vpc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | | [aws_iam_service_linked_role.opensearch](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_service_linked_role) | resource | +| [aws_iam_service_linked_role.opensearchserverless](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_service_linked_role) | resource | | [aws_kinesis_firehose_delivery_stream.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kinesis_firehose_delivery_stream) | resource | | [aws_redshift_cluster_iam_roles.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/redshift_cluster_iam_roles) | resource | | [aws_security_group.destination](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [aws_security_group.firehose](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource | | [aws_security_group_rule.destination](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | | [aws_security_group_rule.firehose](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | -| [aws_security_group_rule.firehose_es_egress_rule](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | +| [aws_security_group_rule.firehose_egress_rule](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group_rule) | resource | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | | [aws_iam_policy_document.application](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.application_assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.assume_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.cross_account_elasticsearch](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.cross_account_opensearch](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.cross_account_opensearchserverless](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.cross_account_s3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.cw](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.elasticsearch](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.glue](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.kinesis](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.lambda](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.opensearch](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | +| [aws_iam_policy_document.opensearchserverless](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.s3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.s3_kms](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_iam_policy_document.vpc](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source | | [aws_region.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/region) | data source | -| [aws_subnet.elasticsearch](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet) | data source | +| [aws_subnet.subnet](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/subnet) | data source | ## Inputs @@ -886,6 +939,7 @@ No modules. | [data\_format\_conversion\_parquet\_writer\_version](#input\_data\_format\_conversion\_parquet\_writer\_version) | Indicates the version of row format to output. | `string` | `"V1"` | no | | [datadog\_endpoint\_type](#input\_datadog\_endpoint\_type) | Endpoint type to datadog destination | `string` | `"logs_eu"` | no | | [destination](#input\_destination) | This is the destination to where the data is delivered | `string` | n/a | yes | +| [destination\_cross\_account](#input\_destination\_cross\_account) | Indicates if destination is in a different account. Only supported to Elasticsearch and OpenSearch | `bool` | `false` | no | | [destination\_log\_group\_name](#input\_destination\_log\_group\_name) | The CloudWatch group name for destination logs | `string` | `null` | no | | [destination\_log\_stream\_name](#input\_destination\_log\_stream\_name) | The CloudWatch log stream name for destination logs | `string` | `null` | no | | [dynamic\_partition\_append\_delimiter\_to\_record](#input\_dynamic\_partition\_append\_delimiter\_to\_record) | To configure your delivery stream to add a new line delimiter between records in objects that are delivered to Amazon S3. | `bool` | `false` | no | @@ -896,18 +950,11 @@ No modules. | [dynamic\_partitioning\_retry\_duration](#input\_dynamic\_partitioning\_retry\_duration) | Total amount of seconds Firehose spends on retries | `number` | `300` | no | | [dynatrace\_api\_url](#input\_dynatrace\_api\_url) | API URL to Dynatrace destination | `string` | `null` | no | | [dynatrace\_endpoint\_location](#input\_dynatrace\_endpoint\_location) | Endpoint Location to Dynatrace destination | `string` | `"eu"` | no | -| [elasticsearch\_cross\_account](#input\_elasticsearch\_cross\_account) | Indicates if Elasticsearch domain is in a different account | `bool` | `false` | no | | [elasticsearch\_domain\_arn](#input\_elasticsearch\_domain\_arn) | The ARN of the Amazon ES domain. The pattern needs to be arn:.* | `string` | `null` | no | -| [elasticsearch\_enable\_vpc](#input\_elasticsearch\_enable\_vpc) | Indicates if destination is configured in VPC. Supported only to Elasticsearch destinations | `bool` | `false` | no | | [elasticsearch\_index\_name](#input\_elasticsearch\_index\_name) | The Elasticsearch index name | `string` | `null` | no | | [elasticsearch\_index\_rotation\_period](#input\_elasticsearch\_index\_rotation\_period) | The Elasticsearch index rotation period. Index rotation appends a timestamp to the IndexName to facilitate expiration of old data | `string` | `"OneDay"` | no | -| [elasticsearch\_retry\_duration](#input\_elasticsearch\_retry\_duration) | The length of time during which Firehose retries delivery after a failure, starting from the initial request and including the first attempt | `string` | `3600` | no | +| [elasticsearch\_retry\_duration](#input\_elasticsearch\_retry\_duration) | The length of time during which Firehose retries delivery after a failure, starting from the initial request and including the first attempt | `string` | `300` | no | | [elasticsearch\_type\_name](#input\_elasticsearch\_type\_name) | The Elasticsearch type name with maximum length of 100 characters | `string` | `null` | no | -| [elasticsearch\_vpc\_create\_service\_linked\_role](#input\_elasticsearch\_vpc\_create\_service\_linked\_role) | Set it to True if want create Opensearch Service Linked Role to Access VPC | `bool` | `false` | no | -| [elasticsearch\_vpc\_role\_arn](#input\_elasticsearch\_vpc\_role\_arn) | The ARN of the IAM role to be assumed by Firehose for calling the Amazon EC2 configuration API and for creating network interfaces | `string` | `null` | no | -| [elasticsearch\_vpc\_security\_group\_same\_as\_destination](#input\_elasticsearch\_vpc\_security\_group\_same\_as\_destination) | Indicates if the firehose security group is the same as destination | `bool` | `true` | no | -| [elasticsearch\_vpc\_subnet\_ids](#input\_elasticsearch\_vpc\_subnet\_ids) | A list of subnet IDs to associate with Kinesis Firehose | `list(string)` | `null` | no | -| [elasticsearch\_vpc\_use\_existing\_role](#input\_elasticsearch\_vpc\_use\_existing\_role) | Indicates if want use the kinesis firehose role to VPC access. | `bool` | `true` | no | | [enable\_data\_format\_conversion](#input\_enable\_data\_format\_conversion) | Set it to true if you want to disable format conversion. | `bool` | `false` | no | | [enable\_destination\_log](#input\_enable\_destination\_log) | The CloudWatch Logging Options for the delivery stream | `bool` | `true` | no | | [enable\_dynamic\_partitioning](#input\_enable\_dynamic\_partitioning) | Enables or disables dynamic partitioning | `bool` | `false` | no | @@ -916,6 +963,7 @@ No modules. | [enable\_s3\_backup](#input\_enable\_s3\_backup) | The Amazon S3 backup mode | `bool` | `false` | no | | [enable\_s3\_encryption](#input\_enable\_s3\_encryption) | Indicates if want use encryption in S3 bucket. | `bool` | `false` | no | | [enable\_sse](#input\_enable\_sse) | Whether to enable encryption at rest. Only makes sense when source is Direct Put | `bool` | `false` | no | +| [enable\_vpc](#input\_enable\_vpc) | Indicates if destination is configured in VPC. Supports Elasticsearch and Opensearch destinations. | `bool` | `false` | no | | [firehose\_role](#input\_firehose\_role) | IAM role ARN attached to the Kinesis Firehose Stream. | `string` | `null` | no | | [honeycomb\_api\_host](#input\_honeycomb\_api\_host) | If you use a Secure Tenancy or other proxy, put its schema://host[:port] here | `string` | `"https://api.honeycomb.io"` | no | | [honeycomb\_dataset\_name](#input\_honeycomb\_dataset\_name) | Your Honeycomb dataset name to Honeycomb destination | `string` | `null` | no | @@ -936,6 +984,15 @@ No modules. | [mongodb\_realm\_webhook\_url](#input\_mongodb\_realm\_webhook\_url) | Realm Webhook URL to use in MongoDB destination | `string` | `null` | no | | [name](#input\_name) | A name to identify the stream. This is unique to the AWS account and region the Stream is created in | `string` | n/a | yes | | [newrelic\_endpoint\_type](#input\_newrelic\_endpoint\_type) | Endpoint type to New Relic destination | `string` | `"logs_eu"` | no | +| [opensearch\_document\_id\_options](#input\_opensearch\_document\_id\_options) | The method for setting up document ID. | `string` | `"FIREHOSE_DEFAULT"` | no | +| [opensearch\_domain\_arn](#input\_opensearch\_domain\_arn) | The ARN of the Amazon Opensearch domain. The pattern needs to be arn:.*. Conflicts with cluster\_endpoint. | `string` | `null` | no | +| [opensearch\_index\_name](#input\_opensearch\_index\_name) | The Opensearch (And OpenSearch Serverless) index name. | `string` | `null` | no | +| [opensearch\_index\_rotation\_period](#input\_opensearch\_index\_rotation\_period) | The Opensearch index rotation period. Index rotation appends a timestamp to the IndexName to facilitate expiration of old data | `string` | `"OneDay"` | no | +| [opensearch\_retry\_duration](#input\_opensearch\_retry\_duration) | After an initial failure to deliver to Amazon OpenSearch, the total amount of time, in seconds between 0 to 7200, during which Firehose re-attempts delivery (including the first attempt). After this time has elapsed, the failed documents are written to Amazon S3. The default value is 300s. There will be no retry if the value is 0. | `string` | `300` | no | +| [opensearch\_type\_name](#input\_opensearch\_type\_name) | The opensearch type name with maximum length of 100 characters. Types are deprecated in OpenSearch\_1.1. TypeName must be empty. | `string` | `null` | no | +| [opensearch\_vpc\_create\_service\_linked\_role](#input\_opensearch\_vpc\_create\_service\_linked\_role) | Set it to True if want create Opensearch Service Linked Role to Access VPC. | `bool` | `false` | no | +| [opensearchserverless\_collection\_arn](#input\_opensearchserverless\_collection\_arn) | The ARN of the Amazon Opensearch Serverless Collection. The pattern needs to be arn:.*. | `string` | `null` | no | +| [opensearchserverless\_collection\_endpoint](#input\_opensearchserverless\_collection\_endpoint) | The endpoint to use when communicating with the collection in the Serverless offering for Amazon OpenSearch Service. | `string` | `null` | no | | [policy\_path](#input\_policy\_path) | Path of policies to that should be added to IAM role for Kinesis Firehose Stream | `string` | `null` | no | | [redshift\_cluster\_endpoint](#input\_redshift\_cluster\_endpoint) | The redshift endpoint | `string` | `null` | no | | [redshift\_cluster\_identifier](#input\_redshift\_cluster\_identifier) | Redshift Cluster identifier. Necessary to associate the iam role to cluster | `string` | `null` | no | @@ -963,7 +1020,7 @@ No modules. | [s3\_backup\_kms\_key\_arn](#input\_s3\_backup\_kms\_key\_arn) | Specifies the KMS key ARN the stream will use to encrypt data. If not set, no encryption will be used. | `string` | `null` | no | | [s3\_backup\_log\_group\_name](#input\_s3\_backup\_log\_group\_name) | he CloudWatch group name for logging | `string` | `null` | no | | [s3\_backup\_log\_stream\_name](#input\_s3\_backup\_log\_stream\_name) | The CloudWatch log stream name for logging | `string` | `null` | no | -| [s3\_backup\_mode](#input\_s3\_backup\_mode) | Defines how documents should be delivered to Amazon S3. Used to elasticsearch, splunk, http configurations. For S3 and Redshift use enable\_s3\_backup | `string` | `"FailedOnly"` | no | +| [s3\_backup\_mode](#input\_s3\_backup\_mode) | Defines how documents should be delivered to Amazon S3. Used to elasticsearch, opensearch, splunk, http configurations. For S3 and Redshift use enable\_s3\_backup | `string` | `"FailedOnly"` | no | | [s3\_backup\_prefix](#input\_s3\_backup\_prefix) | The YYYY/MM/DD/HH time format prefix is automatically used for delivered S3 files. You can specify an extra prefix to be added in front of the time format prefix. Note that if the prefix ends with a slash, it appears as a folder in the S3 bucket | `string` | `null` | no | | [s3\_backup\_role\_arn](#input\_s3\_backup\_role\_arn) | The role that Kinesis Data Firehose can use to access S3 Backup. | `string` | `null` | no | | [s3\_backup\_use\_existing\_role](#input\_s3\_backup\_use\_existing\_role) | Indicates if want use the kinesis firehose role to s3 backup bucket access. | `bool` | `true` | no | @@ -993,12 +1050,16 @@ No modules. | [transform\_lambda\_role\_arn](#input\_transform\_lambda\_role\_arn) | The ARN of the role to execute the transform lambda. If null use the Firehose Stream role | `string` | `null` | no | | [vpc\_create\_destination\_security\_group](#input\_vpc\_create\_destination\_security\_group) | Indicates if want create destination security group to associate to firehose destinations | `bool` | `false` | no | | [vpc\_create\_security\_group](#input\_vpc\_create\_security\_group) | Indicates if want create security group to associate to kinesis firehose | `bool` | `false` | no | +| [vpc\_role\_arn](#input\_vpc\_role\_arn) | The ARN of the IAM role to be assumed by Firehose for calling the Amazon EC2 configuration API and for creating network interfaces. Supports Elasticsearch and Opensearch destinations. | `string` | `null` | no | | [vpc\_security\_group\_destination\_configure\_existing](#input\_vpc\_security\_group\_destination\_configure\_existing) | Indicates if want configure an existing destination security group with the necessary rules | `bool` | `false` | no | | [vpc\_security\_group\_destination\_ids](#input\_vpc\_security\_group\_destination\_ids) | A list of security group IDs associated to destinations to allow firehose traffic | `list(string)` | `null` | no | | [vpc\_security\_group\_destination\_vpc\_id](#input\_vpc\_security\_group\_destination\_vpc\_id) | VPC ID to create the destination security group. Only supported to Redshift and splunk destinations | `string` | `null` | no | | [vpc\_security\_group\_firehose\_configure\_existing](#input\_vpc\_security\_group\_firehose\_configure\_existing) | Indicates if want configure an existing firehose security group with the necessary rules | `bool` | `false` | no | -| [vpc\_security\_group\_firehose\_ids](#input\_vpc\_security\_group\_firehose\_ids) | A list of security group IDs to associate with Kinesis Firehose | `list(string)` | `null` | no | +| [vpc\_security\_group\_firehose\_ids](#input\_vpc\_security\_group\_firehose\_ids) | A list of security group IDs to associate with Kinesis Firehose. | `list(string)` | `null` | no | +| [vpc\_security\_group\_same\_as\_destination](#input\_vpc\_security\_group\_same\_as\_destination) | Indicates if the firehose security group is the same as destination. | `bool` | `true` | no | | [vpc\_security\_group\_tags](#input\_vpc\_security\_group\_tags) | A map of tags to assign to security group | `map(string)` | `{}` | no | +| [vpc\_subnet\_ids](#input\_vpc\_subnet\_ids) | A list of subnet IDs to associate with Kinesis Firehose. Supports Elasticsearch and Opensearch destinations. | `list(string)` | `null` | no | +| [vpc\_use\_existing\_role](#input\_vpc\_use\_existing\_role) | Indicates if want use the kinesis firehose role to VPC access. Supports Elasticsearch and Opensearch destinations. | `bool` | `true` | no | ## Outputs @@ -1011,6 +1072,7 @@ No modules. | [destination\_security\_group\_id](#output\_destination\_security\_group\_id) | Security Group ID associated to destination | | [destination\_security\_group\_name](#output\_destination\_security\_group\_name) | Security Group Name associated to destination | | [destination\_security\_group\_rule\_ids](#output\_destination\_security\_group\_rule\_ids) | Security Group Rules ID created in Destination Security group | +| [elasticsearch\_cross\_account\_service\_policy](#output\_elasticsearch\_cross\_account\_service\_policy) | Elasticsearch Service policy when the opensearch domain belongs to another account | | [firehose\_cidr\_blocks](#output\_firehose\_cidr\_blocks) | Firehose stream cidr blocks to unblock on destination security group | | [firehose\_security\_group\_id](#output\_firehose\_security\_group\_id) | Security Group ID associated to Firehose Stream. Only Supported for elasticsearch destination | | [firehose\_security\_group\_name](#output\_firehose\_security\_group\_name) | Security Group Name associated to Firehose Stream. Only Supported for elasticsearch destination | @@ -1026,8 +1088,10 @@ No modules. | [kinesis\_firehose\_name](#output\_kinesis\_firehose\_name) | The name of the Kinesis Firehose Stream | | [kinesis\_firehose\_role\_arn](#output\_kinesis\_firehose\_role\_arn) | The ARN of the IAM role created for Kinesis Firehose Stream | | [kinesis\_firehose\_version\_id](#output\_kinesis\_firehose\_version\_id) | The Version id of the Kinesis Firehose Stream | -| [opensearch\_cross\_account\_service\_policy](#output\_opensearch\_cross\_account\_service\_policy) | OpenSearch Service policy when the opensearch domain belongs to another account | +| [opensearch\_cross\_account\_service\_policy](#output\_opensearch\_cross\_account\_service\_policy) | Opensearch Service policy when the opensearch domain belongs to another account | | [opensearch\_iam\_service\_linked\_role\_arn](#output\_opensearch\_iam\_service\_linked\_role\_arn) | The ARN of the Opensearch IAM Service linked role | +| [opensearchserverless\_cross\_account\_service\_policy](#output\_opensearchserverless\_cross\_account\_service\_policy) | Opensearch Serverless Service policy when the opensearch domain belongs to another account | +| [opensearchserverless\_iam\_service\_linked\_role\_arn](#output\_opensearchserverless\_iam\_service\_linked\_role\_arn) | The ARN of the Opensearch Serverless IAM Service linked role | | [s3\_cross\_account\_bucket\_policy](#output\_s3\_cross\_account\_bucket\_policy) | Bucket Policy to S3 Bucket Destination when the bucket belongs to another account | @@ -1040,6 +1104,7 @@ No modules. ## Upgrade - Version 1.x to 2.x Upgrade Guide [here](https://github.com/fdmsantos/terraform-aws-kinesis-firehose/blob/main/UPGRADE-2.0.md) +- Version 2.x to 3.x Upgrade Guide [here](https://github.com/fdmsantos/terraform-aws-kinesis-firehose/blob/main/UPGRADE-3.0.md) ## License diff --git a/UPGRADE-3.0.md b/UPGRADE-3.0.md new file mode 100644 index 0000000..ef954a0 --- /dev/null +++ b/UPGRADE-3.0.md @@ -0,0 +1,36 @@ +# Upgrade from v2.x to v3.x + +If you have any questions regarding this upgrade process, please consult the `examples` directory + +If you find a bug, please open an issue with supporting configuration to reproduce. + +## List of backwards incompatible changes + +### New features + +1. New Destination: Opensearch +2. New Destination: Opensearch Serverless + +### Variable and output changes + +1. Renamed variables: + + - `elasticsearch_enable_vpc` -> `enable_vpc` + - `elasticsearch_vpc_use_existing_role` -> `vpc_use_existing_role` + - `elasticsearch_vpc_role_arn` -> `vpc_role_arn` + - `elasticsearch_vpc_subnet_ids` -> `vpc_subnet_ids` + - `elasticsearch_cross_account` -> `destination_cross_account` + - `elasticsearch_vpc_create_service_linked_role` -> `opensearch_vpc_create_service_linked_role` + - `elasticsearch_vpc_security_group_same_as_destination` -> `vpc_security_group_same_as_destination` + +2. Renamed Outputs: + + - `opensearch_cross_account_service_policy` -> `elasticsearch_cross_account_service_policy` + +3. Default Values Changed: + + - `elasticsearch_retry_duration` -> Default value changed from 3600 to 300. If used the default value, please configure this variable equals to 3600 + +4. Added variables: + + - All variables prefixed with opensearch and opensearchserverless diff --git a/examples/elasticsearch/opensearch-in-vpc/main.tf b/examples/elasticsearch/opensearch-in-vpc/main.tf deleted file mode 100644 index 23b7ee9..0000000 --- a/examples/elasticsearch/opensearch-in-vpc/main.tf +++ /dev/null @@ -1,109 +0,0 @@ -resource "random_pet" "this" { - length = 2 -} - -resource "aws_s3_bucket" "s3" { - bucket = "${var.name_prefix}-destination-bucket-${random_pet.this.id}" - force_destroy = true -} - -resource "aws_opensearch_domain" "this" { - domain_name = "firehose-es-cluster" - engine_version = "Elasticsearch_7.10" - cluster_config { - instance_type = "t3.small.search" - instance_count = 1 - } - ebs_options { - ebs_enabled = true - volume_size = 10 - } - advanced_security_options { - enabled = true - internal_user_database_enabled = true - master_user_options { - master_user_name = var.es_username - master_user_password = var.es_password - } - } - encrypt_at_rest { - enabled = true - } - node_to_node_encryption { - enabled = true - } - domain_endpoint_options { - enforce_https = true - tls_security_policy = "Policy-Min-TLS-1-2-2019-07" - } - vpc_options { - security_group_ids = [module.security_groups.destination_security_group_id] - subnet_ids = [module.vpc.private_subnets[0]] - } -} - -resource "aws_kms_key" "this" { - description = "${var.name_prefix}-kms-key" - deletion_window_in_days = 7 -} - -resource "aws_opensearch_domain_policy" "main" { - domain_name = aws_opensearch_domain.this.domain_name - - access_policies = < @@ -56,9 +52,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [es\_password](#input\_es\_password) | ES Password | `string` | `null` | no | -| [es\_username](#input\_es\_username) | ES Username | `string` | `null` | no | -| [name\_prefix](#input\_name\_prefix) | Name prefix to use in resources | `string` | `"firehose-to-public-es"` | no | +| [name\_prefix](#input\_name\_prefix) | Name prefix to use in resources | `string` | `"firehose-to-public-opensearch"` | no | ## Outputs diff --git a/examples/opensearch/direct-put-to-opensearch/main.tf b/examples/opensearch/direct-put-to-opensearch/main.tf new file mode 100644 index 0000000..325110a --- /dev/null +++ b/examples/opensearch/direct-put-to-opensearch/main.tf @@ -0,0 +1,83 @@ +resource "random_pet" "this" { + length = 2 +} + +resource "aws_s3_bucket" "s3" { + bucket = "${var.name_prefix}-dest-bucket-${random_pet.this.id}" + force_destroy = true +} + +resource "aws_opensearch_domain" "this" { + domain_name = "firehose-os-cluster" + engine_version = "Elasticsearch_7.10" + cluster_config { + instance_type = "t3.small.search" + instance_count = 1 + } + ebs_options { + ebs_enabled = true + volume_size = 10 + } + advanced_security_options { + enabled = false + } + + encrypt_at_rest { + enabled = false + } + + node_to_node_encryption { + enabled = false + } + + domain_endpoint_options { + enforce_https = true + tls_security_policy = "Policy-Min-TLS-1-2-2019-07" + } + +} + +resource "aws_kms_key" "this" { + description = "${var.name_prefix}-kms-key" + deletion_window_in_days = 7 +} + +resource "aws_opensearch_domain_policy" "main" { + domain_name = aws_opensearch_domain.this.domain_name + + access_policies = < @@ -49,8 +45,11 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Type | |------|------| | [aws_kms_key.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | -| [aws_opensearch_domain.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/opensearch_domain) | resource | -| [aws_opensearch_domain_policy.main](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/opensearch_domain_policy) | resource | +| [aws_opensearchserverless_access_policy.policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/opensearchserverless_access_policy) | resource | +| [aws_opensearchserverless_collection.os](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/opensearchserverless_collection) | resource | +| [aws_opensearchserverless_security_policy.networking](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/opensearchserverless_security_policy) | resource | +| [aws_opensearchserverless_security_policy.security_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/opensearchserverless_security_policy) | resource | +| [aws_opensearchserverless_vpc_endpoint.vpc_endpoint](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/opensearchserverless_vpc_endpoint) | resource | | [aws_s3_bucket.s3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource | | [random_pet.this](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet) | resource | @@ -58,9 +57,7 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [es\_password](#input\_es\_password) | ES Password | `string` | `null` | no | -| [es\_username](#input\_es\_username) | ES Username | `string` | `null` | no | -| [name\_prefix](#input\_name\_prefix) | Name prefix to use in resources | `string` | `"firehose-to-public-es"` | no | +| [name\_prefix](#input\_name\_prefix) | Name prefix to use in resources | `string` | `"firehose-to-public-opensearch"` | no | | [vpc\_azs](#input\_vpc\_azs) | Redshift AZs | `list(string)` | 
[
  "eu-west-1a",
  "eu-west-1b",
  "eu-west-1c"
]
| no | | [vpc\_cidr](#input\_vpc\_cidr) | VPC CIDR block | `string` | `"10.0.0.0/16"` | no | | [vpc\_private\_subnets](#input\_vpc\_private\_subnets) | VPC Private Subnets | `list(string)` | 
[
  "10.0.1.0/24",
  "10.0.2.0/24",
  "10.0.3.0/24"
]
| no | @@ -70,11 +67,5 @@ Note that this example may create resources which cost money. Run `terraform des | Name | Description | |------|-------------| -| [dashboard\_endpoint](#output\_dashboard\_endpoint) | Kibana Endpoint | -| [destination\_security\_group](#output\_destination\_security\_group) | Destination Security Group | -| [es\_domain](#output\_es\_domain) | Opensearch Domain | -| [es\_endpoint](#output\_es\_endpoint) | Opensearch Endpoint | -| [firehose\_role](#output\_firehose\_role) | Firehose Role | -| [firehose\_security\_group](#output\_firehose\_security\_group) | Firehose Security Group | -| [opensearch\_iam\_service\_linked\_role\_arn](#output\_opensearch\_iam\_service\_linked\_role\_arn) | AWS Opensearch iam Service Linked Role | +| [os\_domain](#output\_os\_domain) | Opensearch Serverless Collection Endpoint | diff --git a/examples/opensearch/direct-put-to-opensearchserverless-in-vpc/main.tf b/examples/opensearch/direct-put-to-opensearchserverless-in-vpc/main.tf new file mode 100644 index 0000000..67f8519 --- /dev/null +++ b/examples/opensearch/direct-put-to-opensearchserverless-in-vpc/main.tf @@ -0,0 +1,163 @@ +locals { + collection_name = "firehose-es-serverless" + index_name = "test" +} + +resource "random_pet" "this" { + length = 2 +} + +resource "aws_s3_bucket" "s3" { + bucket = "${var.name_prefix}-dest-bucket-${random_pet.this.id}" + force_destroy = true +} + +module "vpc" { + source = "terraform-aws-modules/vpc/aws" + name = "${var.name_prefix}-vpc" + cidr = var.vpc_cidr + azs = var.vpc_azs + private_subnets = var.vpc_private_subnets + public_subnets = var.vpc_public_subnets +} + +module "security_groups" { + source = "../../../" + name = "${var.name_prefix}-delivery-stream" + destination = "opensearchserverless" + create = false + vpc_create_security_group = true + vpc_create_destination_security_group = true + vpc_security_group_same_as_destination = false + vpc_security_group_destination_vpc_id = module.vpc.vpc_id +} + +resource "aws_opensearchserverless_vpc_endpoint" "vpc_endpoint" { + name = "example-vpc-endpoint" + vpc_id = module.vpc.vpc_id + subnet_ids = [module.vpc.private_subnets[0]] + security_group_ids = [module.security_groups.destination_security_group_id] +} + +resource "aws_opensearchserverless_security_policy" "security_policy" { + name = "os-security-policy" + type = "encryption" + policy = jsonencode({ + "Rules" = [ + { + "Resource" = [ + "collection/${local.collection_name}" + ], + "ResourceType" = "collection" + } + ], + "AWSOwnedKey" = true + }) +} + +resource "aws_opensearchserverless_security_policy" "networking" { + name = "networking-policy" + type = "network" + description = "Public access" + policy = jsonencode([ + { + Description = "VPC access for collection endpoint", + Rules = [ + { + ResourceType = "collection", + Resource = [ + "collection/${local.collection_name}" + ] + } + ], + AllowFromPublic = false, + SourceVPCEs = [ + aws_opensearchserverless_vpc_endpoint.vpc_endpoint.id + ] + }, + { + Description = "Public access for dashboards", + Rules = [ + { + ResourceType = "dashboard" + Resource = [ + "collection/${local.collection_name}" + ] + } + ], + AllowFromPublic = true + } + ]) +} + +resource "aws_opensearchserverless_access_policy" "policy" { + name = "data-access-policy" + type = "data" + description = "read and write permissions" + policy = jsonencode([{ + Rules = [ + { + ResourceType = "collection", + Resource = [ + "collection/${local.collection_name}" + ], + Permission = [ + "aoss:CreateCollectionItems", + "aoss:DeleteCollectionItems", + "aoss:UpdateCollectionItems", + "aoss:DescribeCollectionItems" + ] + }, + { + ResourceType = "index", + Resource = [ + "index/${local.collection_name}/${local.index_name}" + ], + Permission = [ + "aoss:CreateIndex", + "aoss:DeleteIndex", + "aoss:UpdateIndex", + "aoss:DescribeIndex", + "aoss:ReadDocument", + "aoss:WriteDocument" + ] + } + ], + Principal = [ + module.firehose.kinesis_firehose_role_arn + ], + Description = "Data Access Policy" + }]) +} + +resource "aws_opensearchserverless_collection" "os" { + name = local.collection_name + depends_on = [aws_opensearchserverless_security_policy.security_policy, aws_opensearchserverless_security_policy.networking] +} + +resource "aws_kms_key" "this" { + description = "${var.name_prefix}-kms-key" + deletion_window_in_days = 7 +} + +module "firehose" { + source = "../../../" + name = "${var.name_prefix}-delivery-stream" + destination = "opensearchserverless" + buffering_interval = 60 + opensearchserverless_collection_endpoint = aws_opensearchserverless_collection.os.collection_endpoint + opensearchserverless_collection_arn = aws_opensearchserverless_collection.os.arn + opensearch_vpc_create_service_linked_role = true + opensearch_index_name = local.index_name + enable_vpc = true + vpc_subnet_ids = module.vpc.private_subnets + vpc_security_group_firehose_ids = [module.security_groups.firehose_security_group_id] + s3_backup_mode = "All" + s3_backup_prefix = "backup/" + s3_backup_bucket_arn = aws_s3_bucket.s3.arn + s3_backup_buffering_interval = 100 + s3_backup_buffering_size = 100 + s3_backup_compression = "GZIP" + s3_backup_enable_encryption = true + s3_backup_kms_key_arn = aws_kms_key.this.arn +} diff --git a/examples/opensearch/direct-put-to-opensearchserverless-in-vpc/outputs.tf b/examples/opensearch/direct-put-to-opensearchserverless-in-vpc/outputs.tf new file mode 100644 index 0000000..08e5c92 --- /dev/null +++ b/examples/opensearch/direct-put-to-opensearchserverless-in-vpc/outputs.tf @@ -0,0 +1,4 @@ +output "os_domain" { + description = "Opensearch Serverless Collection Endpoint" + value = aws_opensearchserverless_collection.os.collection_endpoint +} diff --git a/examples/elasticsearch/opensearch-in-vpc/variables.tf b/examples/opensearch/direct-put-to-opensearchserverless-in-vpc/variables.tf similarity index 70% rename from examples/elasticsearch/opensearch-in-vpc/variables.tf rename to examples/opensearch/direct-put-to-opensearchserverless-in-vpc/variables.tf index 57d3a3e..5e3dee7 100644 --- a/examples/elasticsearch/opensearch-in-vpc/variables.tf +++ b/examples/opensearch/direct-put-to-opensearchserverless-in-vpc/variables.tf @@ -1,21 +1,7 @@ variable "name_prefix" { description = "Name prefix to use in resources" type = string - default = "firehose-to-public-es" -} - -variable "es_username" { - description = "ES Username" - type = string - default = null - sensitive = true -} - -variable "es_password" { - description = "ES Password" - type = string - default = null - sensitive = true + default = "firehose-to-public-opensearch" } variable "vpc_cidr" { diff --git a/examples/elasticsearch/public-opensearch/versions.tf b/examples/opensearch/direct-put-to-opensearchserverless-in-vpc/versions.tf similarity index 100% rename from examples/elasticsearch/public-opensearch/versions.tf rename to examples/opensearch/direct-put-to-opensearchserverless-in-vpc/versions.tf diff --git a/examples/opensearch/direct-put-to-opensearchserverless/README.md b/examples/opensearch/direct-put-to-opensearchserverless/README.md new file mode 100644 index 0000000..9440514 --- /dev/null +++ b/examples/opensearch/direct-put-to-opensearchserverless/README.md @@ -0,0 +1,64 @@ +# Public Opensearch Serverless + +Configuration in this directory creates kinesis firehose stream with Direct Put as source and Opensearch Serverless as destination. + +This example can be tested with Demo Data in Kinesis Firehose Console. + +## Usage + +To run this example you need to execute: + +```bash +$ terraform init +$ terraform plan +$ terraform apply +``` + +Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources. + + +## Requirements + +| Name | Version | +|------|---------| +| [terraform](#requirement\_terraform) | >= 0.13.1 | +| [aws](#requirement\_aws) | ~> 5.0 | +| [random](#requirement\_random) | >= 2.0 | + +## Providers + +| Name | Version | +|------|---------| +| [aws](#provider\_aws) | ~> 5.0 | +| [random](#provider\_random) | >= 2.0 | + +## Modules + +| Name | Source | Version | +|------|--------|---------| +| [firehose](#module\_firehose) | ../../../ | n/a | + +## Resources + +| Name | Type | +|------|------| +| [aws_kms_key.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | +| [aws_opensearchserverless_access_policy.policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/opensearchserverless_access_policy) | resource | +| [aws_opensearchserverless_collection.os](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/opensearchserverless_collection) | resource | +| [aws_opensearchserverless_security_policy.networking](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/opensearchserverless_security_policy) | resource | +| [aws_opensearchserverless_security_policy.security_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/opensearchserverless_security_policy) | resource | +| [aws_s3_bucket.s3](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource | +| [random_pet.this](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/pet) | resource | + +## Inputs + +| Name | Description | Type | Default | Required | +|------|-------------|------|---------|:--------:| +| [name\_prefix](#input\_name\_prefix) | Name prefix to use in resources | `string` | `"firehose-to-public-opensearch"` | no | + +## Outputs + +| Name | Description | +|------|-------------| +| [os\_domain](#output\_os\_domain) | Opensearch Serverless Collection Endpoint | + diff --git a/examples/opensearch/direct-put-to-opensearchserverless/main.tf b/examples/opensearch/direct-put-to-opensearchserverless/main.tf new file mode 100644 index 0000000..8a92b3a --- /dev/null +++ b/examples/opensearch/direct-put-to-opensearchserverless/main.tf @@ -0,0 +1,124 @@ +locals { + collection_name = "firehose-es-serverless" + index_name = "test" +} + +resource "random_pet" "this" { + length = 2 +} + +resource "aws_s3_bucket" "s3" { + bucket = "${var.name_prefix}-dest-bucket-${random_pet.this.id}" + force_destroy = true +} + +resource "aws_opensearchserverless_security_policy" "security_policy" { + name = "os-security-policy" + type = "encryption" + policy = jsonencode({ + "Rules" = [ + { + "Resource" = [ + "collection/${local.collection_name}" + ], + "ResourceType" = "collection" + } + ], + "AWSOwnedKey" = true + }) +} + +resource "aws_opensearchserverless_security_policy" "networking" { + name = "networking-policy" + type = "network" + description = "Public access" + policy = jsonencode([ + { + Description = "Public access to collection and Dashboards endpoint for example collection", + Rules = [ + { + ResourceType = "collection", + Resource = [ + "collection/${local.collection_name}" + ] + }, + { + ResourceType = "dashboard" + Resource = [ + "collection/${local.collection_name}" + ] + } + ], + AllowFromPublic = true + } + ]) +} + +resource "aws_opensearchserverless_access_policy" "policy" { + name = "data-access-policy" + type = "data" + description = "read and write permissions" + policy = jsonencode([{ + Rules = [ + { + ResourceType = "collection", + Resource = [ + "collection/${local.collection_name}" + ], + Permission = [ + "aoss:CreateCollectionItems", + "aoss:DeleteCollectionItems", + "aoss:UpdateCollectionItems", + "aoss:DescribeCollectionItems" + ] + }, + { + ResourceType = "index", + Resource = [ + "index/${local.collection_name}/${local.index_name}" + ], + Permission = [ + "aoss:CreateIndex", + "aoss:DeleteIndex", + "aoss:UpdateIndex", + "aoss:DescribeIndex", + "aoss:ReadDocument", + "aoss:WriteDocument" + ] + } + ], + Principal = [ + module.firehose.kinesis_firehose_role_arn + ], + Description = "Data Access Policy" + }]) +} + +resource "aws_opensearchserverless_collection" "os" { + name = local.collection_name + depends_on = [aws_opensearchserverless_security_policy.security_policy, aws_opensearchserverless_security_policy.networking] +} + +resource "aws_kms_key" "this" { + description = "${var.name_prefix}-kms-key" + deletion_window_in_days = 7 +} + +module "firehose" { + source = "../../../" + name = "${var.name_prefix}-delivery-stream" + destination = "opensearchserverless" + buffering_interval = 60 + opensearchserverless_collection_endpoint = aws_opensearchserverless_collection.os.collection_endpoint + opensearchserverless_collection_arn = aws_opensearchserverless_collection.os.arn + opensearch_vpc_create_service_linked_role = true + opensearch_index_name = local.index_name + s3_backup_mode = "All" + s3_backup_prefix = "backup/" + s3_backup_bucket_arn = aws_s3_bucket.s3.arn + s3_backup_buffering_interval = 100 + s3_backup_buffering_size = 100 + s3_backup_compression = "GZIP" + s3_backup_enable_encryption = true + s3_backup_kms_key_arn = aws_kms_key.this.arn +} diff --git a/examples/opensearch/direct-put-to-opensearchserverless/outputs.tf b/examples/opensearch/direct-put-to-opensearchserverless/outputs.tf new file mode 100644 index 0000000..08e5c92 --- /dev/null +++ b/examples/opensearch/direct-put-to-opensearchserverless/outputs.tf @@ -0,0 +1,4 @@ +output "os_domain" { + description = "Opensearch Serverless Collection Endpoint" + value = aws_opensearchserverless_collection.os.collection_endpoint +} diff --git a/examples/opensearch/direct-put-to-opensearchserverless/variables.tf b/examples/opensearch/direct-put-to-opensearchserverless/variables.tf new file mode 100644 index 0000000..9bc7d7e --- /dev/null +++ b/examples/opensearch/direct-put-to-opensearchserverless/variables.tf @@ -0,0 +1,5 @@ +variable "name_prefix" { + description = "Name prefix to use in resources" + type = string + default = "firehose-to-public-opensearch" +} diff --git a/examples/opensearch/direct-put-to-opensearchserverless/versions.tf b/examples/opensearch/direct-put-to-opensearchserverless/versions.tf new file mode 100644 index 0000000..bee969b --- /dev/null +++ b/examples/opensearch/direct-put-to-opensearchserverless/versions.tf @@ -0,0 +1,14 @@ +terraform { + required_version = ">= 0.13.1" + + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + random = { + source = "hashicorp/random" + version = ">= 2.0" + } + } +} diff --git a/iam.tf b/iam.tf index 1c57725..0debbb1 100644 --- a/iam.tf +++ b/iam.tf @@ -1,16 +1,18 @@ locals { - role_name = var.create && var.create_role ? coalesce(var.role_name, var.name, "*") : null - application_role_name = coalesce(var.application_role_name, "${var.name}-application-role", "*") - create_application_role_policy = var.create && var.create_application_role_policy - add_backup_policies = local.enable_s3_backup && var.s3_backup_use_existing_role - add_kinesis_source_policy = var.create && var.create_role && local.is_kinesis_source && var.kinesis_source_use_existing_role - add_lambda_policy = var.create && var.create_role && var.enable_lambda_transform - add_s3_kms_policy = var.create && var.create_role && ((local.add_backup_policies && var.s3_backup_enable_encryption) || var.enable_s3_encryption) - add_glue_policy = var.create && var.create_role && var.enable_data_format_conversion && var.data_format_conversion_glue_use_existing_role - add_s3_policy = var.create && var.create_role - add_cw_policy = var.create && var.create_role && ((local.add_backup_policies && var.s3_backup_enable_log) || var.enable_destination_log) - add_elasticsearch_policy = var.create && var.create_role && local.destination == "elasticsearch" - add_vpc_policy = var.create && var.create_role && var.elasticsearch_enable_vpc && var.elasticsearch_vpc_use_existing_role && local.destination == "elasticsearch" + role_name = var.create && var.create_role ? coalesce(var.role_name, var.name, "*") : null + application_role_name = coalesce(var.application_role_name, "${var.name}-application-role", "*") + create_application_role_policy = var.create && var.create_application_role_policy + add_backup_policies = local.enable_s3_backup && var.s3_backup_use_existing_role + add_kinesis_source_policy = var.create && var.create_role && local.is_kinesis_source && var.kinesis_source_use_existing_role + add_lambda_policy = var.create && var.create_role && var.enable_lambda_transform + add_s3_kms_policy = var.create && var.create_role && ((local.add_backup_policies && var.s3_backup_enable_encryption) || var.enable_s3_encryption) + add_glue_policy = var.create && var.create_role && var.enable_data_format_conversion && var.data_format_conversion_glue_use_existing_role + add_s3_policy = var.create && var.create_role + add_cw_policy = var.create && var.create_role && ((local.add_backup_policies && var.s3_backup_enable_log) || var.enable_destination_log) + add_elasticsearch_policy = var.create && var.create_role && local.destination == "elasticsearch" + add_opensearch_policy = var.create && var.create_role && local.destination == "opensearch" + add_opensearchserverless_policy = var.create && var.create_role && local.destination == "opensearchserverless" + add_vpc_policy = var.create && var.create_role && var.enable_vpc && var.vpc_use_existing_role && local.is_search_destination } data "aws_iam_policy_document" "assume_role" { @@ -371,7 +373,7 @@ resource "aws_iam_role_policy_attachment" "elasticsearch" { } data "aws_iam_policy_document" "cross_account_elasticsearch" { - count = local.add_elasticsearch_policy && var.elasticsearch_cross_account ? 1 : 0 + count = local.add_elasticsearch_policy && var.destination_cross_account ? 1 : 0 version = "2012-10-17" statement { sid = "Cross Account Access to ${data.aws_caller_identity.current.account_id} Account" @@ -398,6 +400,158 @@ data "aws_iam_policy_document" "cross_account_elasticsearch" { ] } } + +################## +# OpenSearch +################## +data "aws_iam_policy_document" "opensearch" { # TODO Change Actions + count = local.add_opensearch_policy ? 1 : 0 + statement { + effect = "Allow" + actions = [ + "es:DescribeDomain", + "es:DescribeDomains", + "es:DescribeDomainConfig", + "es:ESHttpPost", + "es:ESHttpPut" + ] + resources = [ + var.opensearch_domain_arn, + "${var.opensearch_domain_arn}/*" + ] + } + + statement { + effect = "Allow" + actions = [ + "es:ESHttpGet" + ] + resources = [ + "${var.opensearch_domain_arn}/_all/_settings", + "${var.opensearch_domain_arn}/_cluster/stats", + "${var.opensearch_domain_arn}/${var.opensearch_index_name}*/_mapping/${var.opensearch_type_name != null ? var.opensearch_type_name : "*"}", + "${var.opensearch_domain_arn}/_nodes", + "${var.opensearch_domain_arn}/_nodes/stats", + "${var.opensearch_domain_arn}/_nodes/*/stats", + "${var.opensearch_domain_arn}/_stats", + "${var.opensearch_domain_arn}/${var.opensearch_index_name}*/_stats" + ] + } +} + +resource "aws_iam_policy" "opensearch" { + count = local.add_opensearch_policy ? 1 : 0 + + name = "${local.role_name}-opensearch" + path = var.policy_path + policy = data.aws_iam_policy_document.opensearch[0].json + tags = var.tags +} + +resource "aws_iam_role_policy_attachment" "opensearch" { + count = local.add_opensearch_policy ? 1 : 0 + + role = aws_iam_role.firehose[0].name + policy_arn = aws_iam_policy.opensearch[0].arn +} + +data "aws_iam_policy_document" "cross_account_opensearch" { # TODO Change Actions + count = local.add_opensearch_policy && var.destination_cross_account ? 1 : 0 + version = "2012-10-17" + statement { + sid = "Cross Account Access to ${data.aws_caller_identity.current.account_id} Account" + effect = "Allow" + + principals { + type = "AWS" + identifiers = [local.firehose_role_arn] + } + + actions = [ + "es:ESHttpGet" + ] + + resources = [ + "${var.opensearch_domain_arn}/_all/_settings", + "${var.opensearch_domain_arn}/_cluster/stats", + "${var.opensearch_domain_arn}/${var.opensearch_index_name}*/_mapping/${var.opensearch_type_name != null ? var.opensearch_type_name : "*"}", + "${var.opensearch_domain_arn}/_nodes", + "${var.opensearch_domain_arn}/_nodes/stats", + "${var.opensearch_domain_arn}/_nodes/*/stats", + "${var.opensearch_domain_arn}/_stats", + "${var.opensearch_domain_arn}/${var.opensearch_index_name}*/_stats" + ] + } +} # TODO Change Actions + +resource "aws_iam_service_linked_role" "opensearch" { + count = contains(["elasticsearch", "opensearch"], local.destination) && var.enable_vpc && var.opensearch_vpc_create_service_linked_role ? 1 : 0 + aws_service_name = "opensearchservice.amazonaws.com" + description = "Allows Amazon OpenSearch to manage AWS resources for a domain on your behalf." + tags = merge(var.tags, var.role_tags) +} + +################## +# Opensearch Serverless +################## +data "aws_iam_policy_document" "opensearchserverless" { + count = local.add_opensearchserverless_policy ? 1 : 0 + statement { + effect = "Allow" + actions = [ + "aoss:APIAccessAll" + ] + resources = [ + var.opensearchserverless_collection_arn + ] + } +} + +resource "aws_iam_policy" "opensearchserverless" { + count = local.add_opensearchserverless_policy ? 1 : 0 + + name = "${local.role_name}-opensearchserverless" + path = var.policy_path + policy = data.aws_iam_policy_document.opensearchserverless[0].json + tags = var.tags +} + +resource "aws_iam_role_policy_attachment" "opensearchserverless" { + count = local.add_opensearchserverless_policy ? 1 : 0 + + role = aws_iam_role.firehose[0].name + policy_arn = aws_iam_policy.opensearchserverless[0].arn +} + +data "aws_iam_policy_document" "cross_account_opensearchserverless" { + count = local.add_opensearchserverless_policy && var.destination_cross_account ? 1 : 0 + version = "2012-10-17" + statement { + sid = "Cross Account Access to ${data.aws_caller_identity.current.account_id} Account" + effect = "Allow" + + principals { + type = "AWS" + identifiers = [local.firehose_role_arn] + } + + actions = [ + "aoss:APIAccessAll" + ] + + resources = [ + var.opensearchserverless_collection_arn + ] + } +} + +resource "aws_iam_service_linked_role" "opensearchserverless" { + count = local.destination == "opensearchserverless" && var.opensearch_vpc_create_service_linked_role ? 1 : 0 + aws_service_name = "observability.aoss.amazonaws.com" + description = "Allows Amazon OpenSearch Serverless to manage AWS resources for a domain on your behalf." + tags = merge(var.tags, var.role_tags) +} + ################## # VPC ################## @@ -437,14 +591,6 @@ resource "aws_iam_role_policy_attachment" "vpc" { policy_arn = aws_iam_policy.vpc[0].arn } - -resource "aws_iam_service_linked_role" "opensearch" { - count = local.destination == "elasticsearch" && var.elasticsearch_enable_vpc && var.elasticsearch_vpc_create_service_linked_role ? 1 : 0 - aws_service_name = "opensearchservice.amazonaws.com" - description = "Allows Amazon OpenSearch to manage AWS resources for a domain on your behalf." - tags = merge(var.tags, var.role_tags) -} - ################## # Application Role ################## diff --git a/locals.tf b/locals.tf index ca28211..ae32ec4 100644 --- a/locals.tf +++ b/locals.tf @@ -11,7 +11,8 @@ locals { extended_s3 : "extended_s3", redshift : "redshift", elasticsearch : "elasticsearch", - opensearch : "elasticsearch", + opensearch : "opensearch", + opensearchserverless : "opensearchserverless", splunk : "splunk", http_endpoint : "http_endpoint", datadog : "http_endpoint", @@ -23,8 +24,9 @@ locals { mongodb : "http_endpoint", sumologic : "http_endpoint" } - destination = local.destinations[var.destination] - s3_destination = local.destination == "extended_s3" ? true : false + destination = local.destinations[var.destination] + s3_destination = local.destination == "extended_s3" ? true : false + is_search_destination = contains(["elasticsearch", "opensearch", "opensearchserverless"], local.destination) ? true : false # Data Transformation enable_processing = var.enable_lambda_transform || var.enable_dynamic_partitioning @@ -116,7 +118,7 @@ locals { ) : null) # S3 Backup - use_backup_vars_in_s3_configuration = contains(["elasticsearch", "splunk", "http_endpoint"], local.destination) ? true : false + use_backup_vars_in_s3_configuration = contains(["elasticsearch", "opensearch", "opensearchserverless", "splunk", "http_endpoint"], local.destination) ? true : false s3_backup = var.enable_s3_backup ? "Enabled" : "Disabled" enable_s3_backup = var.enable_s3_backup || local.use_backup_vars_in_s3_configuration s3_backup_role_arn = (local.enable_s3_backup ? ( @@ -129,6 +131,14 @@ locals { FailedOnly : "FailedDocumentsOnly", All : "AllDocuments" } + opensearch : { + FailedOnly : "FailedDocumentsOnly", + All : "AllDocuments" + } + opensearchserverless : { + FailedOnly : "FailedDocumentsOnly", + All : "AllDocuments" + } splunk : { FailedOnly : "FailedEventsOnly", All : "AllEvents" @@ -154,21 +164,21 @@ locals { create_backup_logs = var.create && var.enable_s3_backup && var.s3_backup_enable_log && var.s3_backup_create_cw_log_group # VPC Config - elasticsearch_vpc_role_arn = (var.elasticsearch_enable_vpc ? ( - var.elasticsearch_vpc_use_existing_role ? local.firehose_role_arn : var.elasticsearch_vpc_role_arn + vpc_role_arn = (var.enable_vpc ? ( + var.vpc_use_existing_role ? local.firehose_role_arn : var.vpc_role_arn ) : null) - elasticsearch_vpc_create_firehose_sg = local.destination == "elasticsearch" && var.vpc_create_security_group - elasticsearch_vpc_sgs = local.elasticsearch_vpc_create_firehose_sg ? [aws_security_group.firehose[0].id] : var.vpc_security_group_firehose_ids - elasticsearch_vpc_configure_existing_firehose_sg = local.destination == "elasticsearch" && var.elasticsearch_enable_vpc && var.vpc_security_group_firehose_configure_existing && !local.elasticsearch_vpc_create_firehose_sg - elasticsearch_vpc_create_destination_group = local.destination == "elasticsearch" && var.vpc_create_destination_security_group && !var.elasticsearch_vpc_security_group_same_as_destination - elasticsearch_vpc_firehose_sgs = local.elasticsearch_vpc_create_firehose_sg ? [aws_security_group.firehose[0].id] : var.vpc_security_group_firehose_ids - elasticsearch_vpc_destination_sgs = local.elasticsearch_vpc_create_destination_group ? [aws_security_group.destination[0].id] : var.vpc_security_group_destination_ids - not_elasticsearch_vpc_create_destination_group = contains(["splunk", "redshift"], local.destination) && var.vpc_create_destination_security_group - vpc_create_destination_group = local.elasticsearch_vpc_create_destination_group || local.not_elasticsearch_vpc_create_destination_group - elasticsearch_vpc_configure_existing_destination_sg = local.destination == "elasticsearch" && var.elasticsearch_enable_vpc && var.vpc_security_group_destination_configure_existing && !local.elasticsearch_vpc_create_destination_group && !var.elasticsearch_vpc_security_group_same_as_destination - not_elasticsearch_vpc_configure_existing_destination_sg = contains(["splunk", "redshift"], local.destination) && var.vpc_security_group_destination_configure_existing - vpc_configure_destination_group = local.elasticsearch_vpc_configure_existing_destination_sg || local.not_elasticsearch_vpc_configure_existing_destination_sg + search_destination_vpc_create_firehose_sg = local.is_search_destination && var.vpc_create_security_group + search_destination_vpc_sgs = local.search_destination_vpc_create_firehose_sg ? [aws_security_group.firehose[0].id] : var.vpc_security_group_firehose_ids + search_destination_vpc_configure_existing_firehose_sg = local.is_search_destination && var.enable_vpc && var.vpc_security_group_firehose_configure_existing && !local.search_destination_vpc_create_firehose_sg + search_destination_vpc_create_destination_group = local.is_search_destination && var.vpc_create_destination_security_group && !var.vpc_security_group_same_as_destination + search_destination_vpc_firehose_sgs = local.search_destination_vpc_create_firehose_sg ? [aws_security_group.firehose[0].id] : var.vpc_security_group_firehose_ids + search_destination_vpc_destination_sgs = local.search_destination_vpc_create_destination_group ? [aws_security_group.destination[0].id] : var.vpc_security_group_destination_ids + not_search_destination_vpc_create_destination_group = contains(["splunk", "redshift"], local.destination) && var.vpc_create_destination_security_group + vpc_create_destination_group = local.search_destination_vpc_create_destination_group || local.not_search_destination_vpc_create_destination_group + search_destination_vpc_configure_existing_destination_sg = local.is_search_destination && var.enable_vpc && var.vpc_security_group_destination_configure_existing && !local.search_destination_vpc_create_destination_group && !var.vpc_security_group_same_as_destination + not_search_destination_vpc_configure_existing_destination_sg = contains(["splunk", "redshift"], local.destination) && var.vpc_security_group_destination_configure_existing + vpc_configure_destination_group = local.search_destination_vpc_configure_existing_destination_sg || local.not_search_destination_vpc_configure_existing_destination_sg http_endpoint_url = { http_endpoint : var.http_endpoint_url diff --git a/main.tf b/main.tf index 3cc7179..1e8db38 100644 --- a/main.tf +++ b/main.tf @@ -2,9 +2,9 @@ data "aws_caller_identity" "current" {} data "aws_region" "current" {} -data "aws_subnet" "elasticsearch" { - count = local.elasticsearch_vpc_create_firehose_sg && var.elasticsearch_enable_vpc ? 1 : 0 - id = var.elasticsearch_vpc_subnet_ids[0] +data "aws_subnet" "subnet" { + count = local.search_destination_vpc_create_firehose_sg && var.enable_vpc ? 1 : 0 + id = var.vpc_subnet_ids[0] } resource "aws_kinesis_firehose_delivery_stream" "this" { @@ -240,6 +240,129 @@ resource "aws_kinesis_firehose_delivery_stream" "this" { } } + dynamic "splunk_configuration" { + for_each = local.destination == "splunk" ? [1] : [] + content { + hec_endpoint = var.splunk_hec_endpoint + hec_token = var.splunk_hec_token + hec_acknowledgment_timeout = var.splunk_hec_acknowledgment_timeout + hec_endpoint_type = var.splunk_hec_endpoint_type + retry_duration = var.splunk_retry_duration + s3_backup_mode = local.s3_backup_mode + + s3_configuration { + role_arn = !local.use_backup_vars_in_s3_configuration ? local.firehose_role_arn : local.s3_backup_role_arn + bucket_arn = !local.use_backup_vars_in_s3_configuration ? var.s3_bucket_arn : var.s3_backup_bucket_arn + buffering_size = !local.use_backup_vars_in_s3_configuration ? var.s3_configuration_buffering_size : var.s3_backup_buffering_size + buffering_interval = !local.use_backup_vars_in_s3_configuration ? var.s3_configuration_buffering_interval : var.s3_backup_buffering_interval + compression_format = !local.use_backup_vars_in_s3_configuration ? var.s3_compression_format : var.s3_backup_compression + prefix = !local.use_backup_vars_in_s3_configuration ? var.s3_prefix : var.s3_backup_prefix + error_output_prefix = !local.use_backup_vars_in_s3_configuration ? var.s3_error_output_prefix : var.s3_backup_error_output_prefix + kms_key_arn = (!local.use_backup_vars_in_s3_configuration && var.enable_s3_encryption ? var.s3_kms_key_arn : (local.use_backup_vars_in_s3_configuration && var.s3_backup_enable_encryption ? var.s3_backup_kms_key_arn : null)) + } + + dynamic "processing_configuration" { + for_each = local.enable_processing ? [1] : [] + content { + enabled = local.enable_processing + dynamic "processors" { + for_each = local.processors + content { + type = processors.value["type"] + dynamic "parameters" { + for_each = processors.value["parameters"] + content { + parameter_name = parameters.value["name"] + parameter_value = parameters.value["value"] + } + } + } + } + } + } + + dynamic "cloudwatch_logging_options" { + for_each = var.enable_destination_log ? [1] : [] + content { + enabled = var.enable_destination_log + log_group_name = local.destination_cw_log_group_name + log_stream_name = local.destination_cw_log_stream_name + } + } + } + } + + dynamic "http_endpoint_configuration" { + for_each = local.destination == "http_endpoint" ? [1] : [] + content { + url = local.http_endpoint_url[var.destination] + name = local.http_endpoint_name[var.destination] + access_key = var.destination != "sumologic" ? var.http_endpoint_access_key : null + buffering_size = var.buffering_size + buffering_interval = var.buffering_interval + role_arn = local.firehose_role_arn + s3_backup_mode = local.s3_backup_mode + retry_duration = var.http_endpoint_retry_duration + + s3_configuration { + role_arn = !local.use_backup_vars_in_s3_configuration ? local.firehose_role_arn : local.s3_backup_role_arn + bucket_arn = !local.use_backup_vars_in_s3_configuration ? var.s3_bucket_arn : var.s3_backup_bucket_arn + buffering_size = !local.use_backup_vars_in_s3_configuration ? var.s3_configuration_buffering_size : var.s3_backup_buffering_size + buffering_interval = !local.use_backup_vars_in_s3_configuration ? var.s3_configuration_buffering_interval : var.s3_backup_buffering_interval + compression_format = !local.use_backup_vars_in_s3_configuration ? var.s3_compression_format : var.s3_backup_compression + prefix = !local.use_backup_vars_in_s3_configuration ? var.s3_prefix : var.s3_backup_prefix + error_output_prefix = !local.use_backup_vars_in_s3_configuration ? var.s3_error_output_prefix : var.s3_backup_error_output_prefix + kms_key_arn = (!local.use_backup_vars_in_s3_configuration && var.enable_s3_encryption ? var.s3_kms_key_arn : (local.use_backup_vars_in_s3_configuration && var.s3_backup_enable_encryption ? var.s3_backup_kms_key_arn : null)) + } + + dynamic "request_configuration" { + for_each = var.http_endpoint_enable_request_configuration ? [1] : [] + content { + content_encoding = var.http_endpoint_request_configuration_content_encoding + + dynamic "common_attributes" { + for_each = concat(var.http_endpoint_request_configuration_common_attributes, try(local.http_endpoint_destinations_parameters[var.destination], [])) + content { + name = common_attributes.value.name + value = common_attributes.value.value + } + } + + } + } + + dynamic "processing_configuration" { + for_each = local.enable_processing ? [1] : [] + content { + enabled = local.enable_processing + dynamic "processors" { + for_each = local.processors + content { + type = processors.value["type"] + dynamic "parameters" { + for_each = processors.value["parameters"] + content { + parameter_name = parameters.value["name"] + parameter_value = parameters.value["value"] + } + } + } + } + } + } + + dynamic "cloudwatch_logging_options" { + for_each = var.enable_destination_log ? [1] : [] + content { + enabled = var.enable_destination_log + log_group_name = local.destination_cw_log_group_name + log_stream_name = local.destination_cw_log_stream_name + } + } + + } + } + dynamic "elasticsearch_configuration" { for_each = local.destination == "elasticsearch" ? [1] : [] content { @@ -294,26 +417,29 @@ resource "aws_kinesis_firehose_delivery_stream" "this" { } dynamic "vpc_config" { - for_each = var.elasticsearch_enable_vpc ? [1] : [] + for_each = var.enable_vpc ? [1] : [] content { - role_arn = local.elasticsearch_vpc_role_arn - subnet_ids = var.elasticsearch_vpc_subnet_ids - security_group_ids = local.elasticsearch_vpc_sgs + role_arn = local.vpc_role_arn + subnet_ids = var.vpc_subnet_ids + security_group_ids = local.search_destination_vpc_sgs } } } } - dynamic "splunk_configuration" { - for_each = local.destination == "splunk" ? [1] : [] + dynamic "opensearch_configuration" { + for_each = local.destination == "opensearch" ? [1] : [] content { - hec_endpoint = var.splunk_hec_endpoint - hec_token = var.splunk_hec_token - hec_acknowledgment_timeout = var.splunk_hec_acknowledgment_timeout - hec_endpoint_type = var.splunk_hec_endpoint_type - retry_duration = var.splunk_retry_duration - s3_backup_mode = local.s3_backup_mode + domain_arn = var.opensearch_domain_arn + role_arn = local.firehose_role_arn + index_name = var.opensearch_index_name + index_rotation_period = var.opensearch_index_rotation_period + retry_duration = var.opensearch_retry_duration + type_name = var.opensearch_type_name + buffering_interval = var.buffering_interval + buffering_size = var.buffering_size + s3_backup_mode = local.s3_backup_mode s3_configuration { role_arn = !local.use_backup_vars_in_s3_configuration ? local.firehose_role_arn : local.s3_backup_role_arn @@ -354,20 +480,34 @@ resource "aws_kinesis_firehose_delivery_stream" "this" { log_stream_name = local.destination_cw_log_stream_name } } + + dynamic "vpc_config" { + for_each = var.enable_vpc ? [1] : [] + content { + role_arn = local.vpc_role_arn + subnet_ids = var.vpc_subnet_ids + security_group_ids = local.search_destination_vpc_sgs + } + } + + document_id_options { + default_document_id_format = var.opensearch_document_id_options + } + } } - dynamic "http_endpoint_configuration" { - for_each = local.destination == "http_endpoint" ? [1] : [] + dynamic "opensearchserverless_configuration" { + for_each = local.destination == "opensearchserverless" ? [1] : [] content { - url = local.http_endpoint_url[var.destination] - name = local.http_endpoint_name[var.destination] - access_key = var.destination != "sumologic" ? var.http_endpoint_access_key : null - buffering_size = var.buffering_size - buffering_interval = var.buffering_interval - role_arn = local.firehose_role_arn - s3_backup_mode = local.s3_backup_mode - retry_duration = var.http_endpoint_retry_duration + + collection_endpoint = var.opensearchserverless_collection_endpoint + index_name = var.opensearch_index_name + buffering_interval = var.buffering_interval + buffering_size = var.buffering_size + retry_duration = var.opensearch_retry_duration + role_arn = local.firehose_role_arn + s3_backup_mode = local.s3_backup_mode s3_configuration { role_arn = !local.use_backup_vars_in_s3_configuration ? local.firehose_role_arn : local.s3_backup_role_arn @@ -380,22 +520,6 @@ resource "aws_kinesis_firehose_delivery_stream" "this" { kms_key_arn = (!local.use_backup_vars_in_s3_configuration && var.enable_s3_encryption ? var.s3_kms_key_arn : (local.use_backup_vars_in_s3_configuration && var.s3_backup_enable_encryption ? var.s3_backup_kms_key_arn : null)) } - dynamic "request_configuration" { - for_each = var.http_endpoint_enable_request_configuration ? [1] : [] - content { - content_encoding = var.http_endpoint_request_configuration_content_encoding - - dynamic "common_attributes" { - for_each = concat(var.http_endpoint_request_configuration_common_attributes, try(local.http_endpoint_destinations_parameters[var.destination], [])) - content { - name = common_attributes.value.name - value = common_attributes.value.value - } - } - - } - } - dynamic "processing_configuration" { for_each = local.enable_processing ? [1] : [] content { @@ -425,6 +549,15 @@ resource "aws_kinesis_firehose_delivery_stream" "this" { } } + dynamic "vpc_config" { + for_each = var.enable_vpc ? [1] : [] + content { + role_arn = local.vpc_role_arn + subnet_ids = var.vpc_subnet_ids + security_group_ids = local.search_destination_vpc_sgs + } + } + } } @@ -458,13 +591,13 @@ resource "aws_cloudwatch_log_stream" "destination" { # Security Group ################## resource "aws_security_group" "firehose" { - count = local.elasticsearch_vpc_create_firehose_sg ? 1 : 0 + count = local.search_destination_vpc_create_firehose_sg ? 1 : 0 name = "${var.name}-sg" - description = !var.elasticsearch_vpc_security_group_same_as_destination ? "Security group to kinesis firehose" : "Security Group to kinesis firehose and destination" - vpc_id = var.elasticsearch_enable_vpc ? data.aws_subnet.elasticsearch[0].vpc_id : var.vpc_security_group_destination_vpc_id + description = !var.vpc_security_group_same_as_destination ? "Security group to kinesis firehose" : "Security Group to kinesis firehose and destination" + vpc_id = var.enable_vpc ? data.aws_subnet.subnet[0].vpc_id : var.vpc_security_group_destination_vpc_id dynamic "ingress" { - for_each = var.elasticsearch_vpc_security_group_same_as_destination ? [1] : [] + for_each = var.vpc_security_group_same_as_destination ? [1] : [] content { from_port = 443 to_port = 443 @@ -477,8 +610,8 @@ resource "aws_security_group" "firehose" { tags = merge(var.tags, var.vpc_security_group_tags) } -resource "aws_security_group_rule" "firehose_es_egress_rule" { - for_each = local.elasticsearch_vpc_create_firehose_sg && !var.elasticsearch_vpc_security_group_same_as_destination ? (local.vpc_create_destination_group ? { for key, value in [aws_security_group.destination[0].id] : key => value } : { for key, value in var.vpc_security_group_destination_ids : key => value }) : {} +resource "aws_security_group_rule" "firehose_egress_rule" { + for_each = local.search_destination_vpc_create_firehose_sg && !var.vpc_security_group_same_as_destination ? (local.vpc_create_destination_group ? { for key, value in [aws_security_group.destination[0].id] : key => value } : { for key, value in var.vpc_security_group_destination_ids : key => value }) : {} type = "egress" from_port = 443 to_port = 443 @@ -492,21 +625,21 @@ resource "aws_security_group" "destination" { count = local.vpc_create_destination_group ? 1 : 0 name = "${var.name}-destination-sg" description = "Allow Inbound traffic from kinesis firehose stream" - vpc_id = local.destination == "elasticsearch" && var.elasticsearch_enable_vpc ? data.aws_subnet.elasticsearch[0].vpc_id : var.vpc_security_group_destination_vpc_id + vpc_id = local.is_search_destination && var.enable_vpc ? data.aws_subnet.subnet[0].vpc_id : var.vpc_security_group_destination_vpc_id dynamic "ingress" { - for_each = local.destination == "elasticsearch" ? [1] : [] + for_each = local.is_search_destination ? [1] : [] content { from_port = 443 to_port = 443 protocol = "tcp" - security_groups = local.elasticsearch_vpc_sgs + security_groups = local.search_destination_vpc_sgs description = "Allow inbound traffic from Kinesis Firehose" } } dynamic "ingress" { - for_each = local.destination != "elasticsearch" ? [1] : [] + for_each = !local.is_search_destination ? [1] : [] content { from_port = 443 to_port = 443 @@ -520,25 +653,25 @@ resource "aws_security_group" "destination" { } resource "aws_security_group_rule" "firehose" { - for_each = local.elasticsearch_vpc_configure_existing_firehose_sg ? (var.elasticsearch_vpc_security_group_same_as_destination ? toset(var.vpc_security_group_firehose_ids) : toset(flatten([for security_group in var.vpc_security_group_firehose_ids : [for destination_sg in local.elasticsearch_vpc_destination_sgs : "${security_group}_${destination_sg}"]]))) : toset([]) - type = var.elasticsearch_vpc_security_group_same_as_destination ? "ingress" : "egress" + for_each = local.search_destination_vpc_configure_existing_firehose_sg ? (var.vpc_security_group_same_as_destination ? toset(var.vpc_security_group_firehose_ids) : toset(flatten([for security_group in var.vpc_security_group_firehose_ids : [for destination_sg in local.search_destination_vpc_destination_sgs : "${security_group}_${destination_sg}"]]))) : toset([]) + type = var.vpc_security_group_same_as_destination ? "ingress" : "egress" from_port = 443 to_port = 443 protocol = "tcp" - security_group_id = var.elasticsearch_vpc_security_group_same_as_destination ? each.value : split("_", each.value)[0] - source_security_group_id = !var.elasticsearch_vpc_security_group_same_as_destination ? split("_", each.value)[1] : null - self = var.elasticsearch_vpc_security_group_same_as_destination ? true : false - description = var.elasticsearch_vpc_security_group_same_as_destination ? "Allow Inbound HTTPS Traffic" : "Allow Outbound HTTPS Traffic" + security_group_id = var.vpc_security_group_same_as_destination ? each.value : split("_", each.value)[0] + source_security_group_id = !var.vpc_security_group_same_as_destination ? split("_", each.value)[1] : null + self = var.vpc_security_group_same_as_destination ? true : false + description = var.vpc_security_group_same_as_destination ? "Allow Inbound HTTPS Traffic" : "Allow Outbound HTTPS Traffic" } resource "aws_security_group_rule" "destination" { - for_each = local.vpc_configure_destination_group ? (local.destination == "elasticsearch" ? flatten([for security_group in var.vpc_security_group_destination_ids : [for destination_sg in local.elasticsearch_vpc_firehose_sgs : "${security_group}_${destination_sg}"]]) : { for key, value in var.vpc_security_group_destination_ids : key => value }) : {} + for_each = local.vpc_configure_destination_group ? (local.is_search_destination ? flatten([for security_group in var.vpc_security_group_destination_ids : [for destination_sg in local.search_destination_vpc_firehose_sgs : "${security_group}_${destination_sg}"]]) : { for key, value in var.vpc_security_group_destination_ids : key => value }) : {} type = "ingress" from_port = 443 to_port = 443 protocol = "tcp" - cidr_blocks = local.destination != "elasticsearch" ? local.firehose_cidr_blocks[local.destination][data.aws_region.current.name] : null - security_group_id = local.destination == "elasticsearch" ? split("_", each.value)[0] : each.value - source_security_group_id = local.destination == "elasticsearch" ? split("_", each.value)[1] : null + cidr_blocks = !local.is_search_destination ? local.firehose_cidr_blocks[local.destination][data.aws_region.current.name] : null + security_group_id = local.is_search_destination ? split("_", each.value)[0] : each.value + source_security_group_id = local.is_search_destination ? split("_", each.value)[1] : null description = "Allow Inbound HTTPS Traffic from Firehose" } diff --git a/outputs.tf b/outputs.tf index 614b1df..878215f 100644 --- a/outputs.tf +++ b/outputs.tf @@ -66,11 +66,27 @@ output "opensearch_iam_service_linked_role_arn" { value = try(aws_iam_service_linked_role.opensearch[0].arn, "") } -output "opensearch_cross_account_service_policy" { - description = "OpenSearch Service policy when the opensearch domain belongs to another account" +output "elasticsearch_cross_account_service_policy" { + description = "Elasticsearch Service policy when the opensearch domain belongs to another account" value = try(data.aws_iam_policy_document.cross_account_elasticsearch[0].json, "") } +output "opensearch_cross_account_service_policy" { + description = "Opensearch Service policy when the opensearch domain belongs to another account" + value = try(data.aws_iam_policy_document.cross_account_opensearch[0].json, "") +} + +output "opensearchserverless_cross_account_service_policy" { + description = "Opensearch Serverless Service policy when the opensearch domain belongs to another account" + value = try(data.aws_iam_policy_document.cross_account_opensearchserverless[0].json, "") +} + + +output "opensearchserverless_iam_service_linked_role_arn" { + description = "The ARN of the Opensearch Serverless IAM Service linked role" + value = try(aws_iam_service_linked_role.opensearchserverless[0].arn, "") +} + output "application_role_arn" { description = "The ARN of the IAM role created for Kinesis Firehose Stream Source" value = try(aws_iam_role.application[0].arn, "") @@ -94,12 +110,12 @@ output "application_role_policy_name" { # Security Group output "firehose_security_group_id" { description = "Security Group ID associated to Firehose Stream. Only Supported for elasticsearch destination" - value = local.elasticsearch_vpc_create_firehose_sg ? aws_security_group.firehose[0].id : null + value = local.search_destination_vpc_create_firehose_sg ? aws_security_group.firehose[0].id : null } output "firehose_security_group_name" { description = "Security Group Name associated to Firehose Stream. Only Supported for elasticsearch destination" - value = local.elasticsearch_vpc_create_firehose_sg ? aws_security_group.firehose[0].name : null + value = local.search_destination_vpc_create_firehose_sg ? aws_security_group.firehose[0].name : null } output "destination_security_group_id" { @@ -114,7 +130,7 @@ output "destination_security_group_name" { output "firehose_security_group_rule_ids" { description = "Security Group Rules ID created in Firehose Stream Security group. Only Supported for elasticsearch destination" - value = local.elasticsearch_vpc_configure_existing_firehose_sg ? aws_security_group_rule.firehose[*].id : null + value = local.search_destination_vpc_configure_existing_firehose_sg ? aws_security_group_rule.firehose[*].id : null } output "destination_security_group_rule_ids" { diff --git a/variables.tf b/variables.tf index 57aad96..efb6737 100644 --- a/variables.tf +++ b/variables.tf @@ -24,7 +24,7 @@ variable "destination" { type = string validation { error_message = "Please use a valid destination!" - condition = contains(["s3", "extended_s3", "redshift", "opensearch", "elasticsearch", "splunk", "http_endpoint", "datadog", "coralogix", "newrelic", "dynatrace", "honeycomb", "logicmonitor", "mongodb", "sumologic"], var.destination) + condition = contains(["s3", "extended_s3", "redshift", "opensearch", "opensearchserverless", "elasticsearch", "splunk", "http_endpoint", "datadog", "coralogix", "newrelic", "dynatrace", "honeycomb", "logicmonitor", "mongodb", "sumologic"], var.destination) } } @@ -233,7 +233,7 @@ variable "s3_backup_log_stream_name" { } variable "s3_backup_mode" { - description = "Defines how documents should be delivered to Amazon S3. Used to elasticsearch, splunk, http configurations. For S3 and Redshift use enable_s3_backup" + description = "Defines how documents should be delivered to Amazon S3. Used to elasticsearch, opensearch, splunk, http configurations. For S3 and Redshift use enable_s3_backup" type = string default = "FailedOnly" validation { @@ -330,6 +330,12 @@ variable "s3_cross_account" { default = false } +variable "destination_cross_account" { + description = "Indicates if destination is in a different account. Only supported to Elasticsearch and OpenSearch" + type = bool + default = false +} + ###### # Kinesis Source ###### @@ -773,45 +779,116 @@ variable "elasticsearch_type_name" { variable "elasticsearch_retry_duration" { description = "The length of time during which Firehose retries delivery after a failure, starting from the initial request and including the first attempt" type = string - default = 3600 + default = 300 + validation { + error_message = "Minimum: 0 seconds." + condition = var.elasticsearch_retry_duration >= 0 && var.elasticsearch_retry_duration <= 7200 + } +} + +###### +# Opensearch Destination Variables +###### +variable "opensearch_domain_arn" { + description = "The ARN of the Amazon Opensearch domain. The pattern needs to be arn:.*. Conflicts with cluster_endpoint." + type = string + default = null +} + +variable "opensearch_index_name" { + description = "The Opensearch (And OpenSearch Serverless) index name." + type = string + default = null +} + +variable "opensearch_index_rotation_period" { + description = "The Opensearch index rotation period. Index rotation appends a timestamp to the IndexName to facilitate expiration of old data" + type = string + default = "OneDay" + validation { + error_message = "Valid values are NoRotation, OneHour, OneDay, OneWeek, and OneMonth." + condition = contains(["NoRotation", "OneHour", "OneDay", "OneWeek", "OneMonth"], var.opensearch_index_rotation_period) + } +} + +variable "opensearch_type_name" { + description = "The opensearch type name with maximum length of 100 characters. Types are deprecated in OpenSearch_1.1. TypeName must be empty." + type = string + default = null +} + +variable "opensearch_retry_duration" { + description = "After an initial failure to deliver to Amazon OpenSearch, the total amount of time, in seconds between 0 to 7200, during which Firehose re-attempts delivery (including the first attempt). After this time has elapsed, the failed documents are written to Amazon S3. The default value is 300s. There will be no retry if the value is 0." + type = string + default = 300 validation { error_message = "Minimum: 0 seconds." - condition = var.elasticsearch_retry_duration >= 0 + condition = var.opensearch_retry_duration >= 0 && var.opensearch_retry_duration <= 7200 } } -variable "elasticsearch_enable_vpc" { - description = "Indicates if destination is configured in VPC. Supported only to Elasticsearch destinations" +variable "opensearch_vpc_create_service_linked_role" { + description = "Set it to True if want create Opensearch Service Linked Role to Access VPC." type = bool default = false } -variable "elasticsearch_vpc_use_existing_role" { - description = "Indicates if want use the kinesis firehose role to VPC access." +variable "opensearch_document_id_options" { + description = "The method for setting up document ID." + type = string + default = "FIREHOSE_DEFAULT" + validation { + error_message = "Valid values are FIREHOSE_DEFAULT and NO_DOCUMENT_ID." + condition = contains(["FIREHOSE_DEFAULT", "NO_DOCUMENT_ID"], var.opensearch_document_id_options) + } +} + +variable "opensearchserverless_collection_endpoint" { + description = "The endpoint to use when communicating with the collection in the Serverless offering for Amazon OpenSearch Service." + type = string + default = null +} + +variable "opensearchserverless_collection_arn" { + description = "The ARN of the Amazon Opensearch Serverless Collection. The pattern needs to be arn:.*." + type = string + default = null +} +###### +# VPC Variables +###### +variable "enable_vpc" { + description = "Indicates if destination is configured in VPC. Supports Elasticsearch and Opensearch destinations." + type = bool + default = false +} + +variable "vpc_use_existing_role" { + description = "Indicates if want use the kinesis firehose role to VPC access. Supports Elasticsearch and Opensearch destinations." type = bool default = true } -variable "elasticsearch_vpc_role_arn" { - description = "The ARN of the IAM role to be assumed by Firehose for calling the Amazon EC2 configuration API and for creating network interfaces" +variable "vpc_role_arn" { + description = "The ARN of the IAM role to be assumed by Firehose for calling the Amazon EC2 configuration API and for creating network interfaces. Supports Elasticsearch and Opensearch destinations." type = string default = null } -variable "elasticsearch_vpc_subnet_ids" { - description = "A list of subnet IDs to associate with Kinesis Firehose" +variable "vpc_subnet_ids" { + description = "A list of subnet IDs to associate with Kinesis Firehose. Supports Elasticsearch and Opensearch destinations." type = list(string) default = null } -variable "elasticsearch_cross_account" { - description = "Indicates if Elasticsearch domain is in a different account" +variable "vpc_security_group_same_as_destination" { + description = "Indicates if the firehose security group is the same as destination." type = bool - default = false + default = true } variable "vpc_security_group_firehose_ids" { - description = "A list of security group IDs to associate with Kinesis Firehose" + description = "A list of security group IDs to associate with Kinesis Firehose." type = list(string) default = null } @@ -828,18 +905,6 @@ variable "vpc_security_group_firehose_configure_existing" { default = false } -variable "elasticsearch_vpc_security_group_same_as_destination" { - description = "Indicates if the firehose security group is the same as destination" - type = bool - default = true -} - -variable "elasticsearch_vpc_create_service_linked_role" { - description = "Set it to True if want create Opensearch Service Linked Role to Access VPC" - type = bool - default = false -} - variable "vpc_security_group_tags" { description = "A map of tags to assign to security group" type = map(string) diff --git a/versions.tf b/versions.tf index b2a7dd2..60d735e 100644 --- a/versions.tf +++ b/versions.tf @@ -3,8 +3,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 5.0" + version = ">= 5.33" } } - }