From eb7f838125b7d4771e6ab071c8112aac27d8bb84 Mon Sep 17 00:00:00 2001 From: Benjamin Edwards Date: Thu, 7 Dec 2023 19:24:03 -0500 Subject: [PATCH] firehose addon module updates (#15439) --- changes/issue-15438-firehose-addon | 1 + .../target-account/.header.md | 9 +++ .../target-account/.terraform-docs.yml | 1 + .../target-account/README.md | 31 ++++---- .../target-account/firehose.tf | 76 +++++++------------ .../target-account/iam.tf | 58 ++++---------- .../target-account/outputs.tf | 9 +-- .../target-account/s3.tf | 7 +- .../target-account/variables.tf | 71 ++++++++++------- .../target-account/version.tf | 2 +- 10 files changed, 113 insertions(+), 152 deletions(-) create mode 100644 changes/issue-15438-firehose-addon create mode 100644 terraform/addons/byo-firehose-logging-destination/target-account/.header.md create mode 100644 terraform/addons/byo-firehose-logging-destination/target-account/.terraform-docs.yml diff --git a/changes/issue-15438-firehose-addon b/changes/issue-15438-firehose-addon new file mode 100644 index 000000000000..aa628074d854 --- /dev/null +++ b/changes/issue-15438-firehose-addon @@ -0,0 +1 @@ +* Update firehose delivery addon to use latest module version, this includes breaking changes to previous configurations as the default prefixes have been changed to natively support time-partitioned Athena table creation \ No newline at end of file diff --git a/terraform/addons/byo-firehose-logging-destination/target-account/.header.md b/terraform/addons/byo-firehose-logging-destination/target-account/.header.md new file mode 100644 index 000000000000..8e754c402a0f --- /dev/null +++ b/terraform/addons/byo-firehose-logging-destination/target-account/.header.md @@ -0,0 +1,9 @@ +# Firehose Logging Destination Setup + +In this Terraform code, we are defining an IAM Role named `fleet_role` in our AWS Account, that will be assumed by the Fleet application we are hosting. We are only allowing this specific IAM Role (identified by its ARN) to perform certain actions on the Firehose service, such as `DescribeDeliveryStream`, `PutRecord`, and `PutRecordBatch`. + +The reason we need a local IAM role in your account is so that we can assume role into it, and you have full control over the permissions it has. The associated IAM policy in the same file specifies the minimum allowed permissions. + +The Firehose service is KMS encrypted, so the IAM Role we assume into needs permission to the KMS key that is being used to encrypt the data going into Firehose. Additionally, if the data is being delivered to S3, it will also be encrypted with KMS using the AWS S3 KMS key that is managed by AWS. This is because only customer managed keys can be shared across accounts, and the Firehose delivery stream is actually the one writing to S3. + +This code sets up a secure and controlled environment for the Fleet application to perform its necessary actions on the Firehose service within your AWS Account. \ No newline at end of file diff --git a/terraform/addons/byo-firehose-logging-destination/target-account/.terraform-docs.yml b/terraform/addons/byo-firehose-logging-destination/target-account/.terraform-docs.yml new file mode 100644 index 000000000000..1d139ddb401d --- /dev/null +++ b/terraform/addons/byo-firehose-logging-destination/target-account/.terraform-docs.yml @@ -0,0 +1 @@ +header-from: .header.md diff --git a/terraform/addons/byo-firehose-logging-destination/target-account/README.md b/terraform/addons/byo-firehose-logging-destination/target-account/README.md index c492c352a736..fe91b4f304d5 100644 --- a/terraform/addons/byo-firehose-logging-destination/target-account/README.md +++ b/terraform/addons/byo-firehose-logging-destination/target-account/README.md @@ -1,4 +1,4 @@ -## Introduction +# Firehose Logging Destination Setup In this Terraform code, we are defining an IAM Role named `fleet_role` in our AWS Account, that will be assumed by the Fleet application we are hosting. We are only allowing this specific IAM Role (identified by its ARN) to perform certain actions on the Firehose service, such as `DescribeDeliveryStream`, `PutRecord`, and `PutRecordBatch`. @@ -6,20 +6,20 @@ The reason we need a local IAM role in your account is so that we can assume rol The Firehose service is KMS encrypted, so the IAM Role we assume into needs permission to the KMS key that is being used to encrypt the data going into Firehose. Additionally, if the data is being delivered to S3, it will also be encrypted with KMS using the AWS S3 KMS key that is managed by AWS. This is because only customer managed keys can be shared across accounts, and the Firehose delivery stream is actually the one writing to S3. -Overall, this code sets up a secure and controlled environment for the Fleet application to perform its necessary actions on the Firehose service within your AWS Account. - +This code sets up a secure and controlled environment for the Fleet application to perform its necessary actions on the Firehose service within your AWS Account. + ## Requirements | Name | Version | |------|---------| | [terraform](#requirement\_terraform) | >= 1.3.7 | -| [aws](#requirement\_aws) | >= 4.52.0 | +| [aws](#requirement\_aws) | >= 5.29.0 | ## Providers | Name | Version | |------|---------| -| [aws](#provider\_aws) | >= 4.52.0 | +| [aws](#provider\_aws) | >= 5.29.0 | ## Modules @@ -30,16 +30,14 @@ No modules. | Name | Type | |------|------| | [aws_iam_policy.firehose](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_iam_policy.fleet-firehose](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | -| [aws_iam_policy_attachment.fleet-firehose](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy_attachment) | resource | +| [aws_iam_policy.fleet_firehose](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource | | [aws_iam_role.firehose](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | | [aws_iam_role.fleet_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource | | [aws_iam_role_policy_attachment.firehose](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | -| [aws_kinesis_firehose_delivery_stream.osquery_results](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kinesis_firehose_delivery_stream) | resource | -| [aws_kinesis_firehose_delivery_stream.osquery_status](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kinesis_firehose_delivery_stream) | resource | -| [aws_kms_key.firehose](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | +| [aws_iam_role_policy_attachment.fleet_firehose](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource | +| [aws_kinesis_firehose_delivery_stream.fleet_log_destinations](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kinesis_firehose_delivery_stream) | resource | +| [aws_kms_key.firehose_key](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource | | [aws_s3_bucket.destination](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource | -| [aws_s3_bucket_acl.destination](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_acl) | resource | | [aws_s3_bucket_public_access_block.destination](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_public_access_block) | resource | | [aws_s3_bucket_server_side_encryption_configuration.destination](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_server_side_encryption_configuration) | resource | | [aws_caller_identity.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/caller_identity) | data source | @@ -54,19 +52,16 @@ No modules. | Name | Description | Type | Default | Required | |------|-------------|------|---------|:--------:| -| [firehose\_results\_name](#input\_firehose\_results\_name) | firehose delivery stream name for osquery results logs | `string` | `"osquery_results"` | no | -| [firehose\_status\_name](#input\_firehose\_status\_name) | firehose delivery stream name for osquery status logs | `string` | `"osquery_status"` | no | | [fleet\_iam\_role\_arn](#input\_fleet\_iam\_role\_arn) | the arn of the fleet role that firehose will assume to write data to your bucket | `string` | n/a | yes | +| [kms\_key\_arn](#input\_kms\_key\_arn) | An optional KMS key ARN for server-side encryption. If not provided and encryption is enabled, a new key will be created. | `string` | `""` | no | +| [log\_destinations](#input\_log\_destinations) | A map of configurations for Firehose delivery streams. | 
map(object({
    name                  = string
    prefix                = string
    error_output_prefix   = string
    buffering_size        = number
    buffering_interval    = number
    compression_format    = string
  }))
| 
{
  "audit": {
    "buffering_interval": 120,
    "buffering_size": 20,
    "compression_format": "UNCOMPRESSED",
    "error_output_prefix": "audit/error/error=!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/",
    "name": "fleet_audit",
    "prefix": "audit/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"
  },
  "results": {
    "buffering_interval": 120,
    "buffering_size": 20,
    "compression_format": "UNCOMPRESSED",
    "error_output_prefix": "results/error/error=!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/",
    "name": "osquery_results",
    "prefix": "results/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"
  },
  "status": {
    "buffering_interval": 120,
    "buffering_size": 20,
    "compression_format": "UNCOMPRESSED",
    "error_output_prefix": "status/error/error=!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/",
    "name": "osquery_status",
    "prefix": "status/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"
  }
}
| no | | [osquery\_logging\_destination\_bucket\_name](#input\_osquery\_logging\_destination\_bucket\_name) | name of the bucket to store osquery results & status logs | `string` | n/a | yes | -| [results\_prefix](#input\_results\_prefix) | s3 object prefix to give to results logs | `string` | `"results/"` | no | -| [status\_prefix](#input\_status\_prefix) | s3 object prefix to give status logs | `string` | `"status/"` | no | +| [server\_side\_encryption\_enabled](#input\_server\_side\_encryption\_enabled) | A boolean flag to enable/disable server-side encryption. Defaults to true (enabled). | `bool` | `true` | no | ## Outputs | Name | Description | |------|-------------| | [firehose\_iam\_role](#output\_firehose\_iam\_role) | n/a | -| [firehose\_results](#output\_firehose\_results) | n/a | -| [firehose\_status](#output\_firehose\_status) | n/a | +| [log\_destinations](#output\_log\_destinations) | Map of Firehose delivery streams' names. | | [s3\_destination](#output\_s3\_destination) | n/a | - \ No newline at end of file diff --git a/terraform/addons/byo-firehose-logging-destination/target-account/firehose.tf b/terraform/addons/byo-firehose-logging-destination/target-account/firehose.tf index 7f9d56096d78..c4b45b7c2f40 100644 --- a/terraform/addons/byo-firehose-logging-destination/target-account/firehose.tf +++ b/terraform/addons/byo-firehose-logging-destination/target-account/firehose.tf @@ -30,13 +30,9 @@ data "aws_iam_policy_document" "firehose_policy" { statement { effect = "Allow" actions = ["logs:PutLogEvents"] - resources = concat([ - "arn:aws:logs:${data.aws_region.current.id}:${data.aws_caller_identity.current.account_id}:log-group:/aws/kinesisfirehose/${var.firehose_results_name}:*", - "arn:aws:logs:${data.aws_region.current.id}:${data.aws_caller_identity.current.account_id}:log-group:/aws/kinesisfirehose/${var.firehose_status_name}:*", - ], - var.firehose_status_name == "" ? [] : [ - "arn:aws:logs:${data.aws_region.current.id}:${data.aws_caller_identity.current.account_id}:log-group:/aws/kinesisfirehose/${var.firehose_audit_name}:*" - ]) + resources = [ + for name in keys(var.log_destinations) : "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:/aws/kinesisfirehose/${var.log_destinations[name].name}:*" + ] } statement { @@ -63,52 +59,32 @@ resource "aws_iam_role_policy_attachment" "firehose" { role = aws_iam_role.firehose.name } -resource "aws_kms_key" "firehose" { - enable_key_rotation = true -} - -resource "aws_kinesis_firehose_delivery_stream" "osquery_results" { - name = var.firehose_results_name - destination = "s3" - - server_side_encryption { - key_arn = aws_kms_key.firehose.arn - } - - s3_configuration { - prefix = var.results_prefix - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.destination.arn - } -} - -resource "aws_kinesis_firehose_delivery_stream" "osquery_status" { - name = var.firehose_status_name - destination = "s3" - - server_side_encryption { - key_arn = aws_kms_key.firehose.arn - } - - s3_configuration { - prefix = var.status_prefix - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.destination.arn - } +resource "aws_kms_key" "firehose_key" { + count = var.server_side_encryption_enabled && length(var.kms_key_arn) == 0 ? 1 : 0 + description = "KMS key for encrypting Firehose data." } -resource "aws_kinesis_firehose_delivery_stream" "fleet_audit" { - count = length(var.firehose_audit_name) > 0 ? 1 : 0 - name = var.firehose_audit_name - destination = "s3" - - server_side_encryption { - key_arn = aws_kms_key.firehose.arn +resource "aws_kinesis_firehose_delivery_stream" "fleet_log_destinations" { + for_each = var.log_destinations + name = each.value.name + destination = "extended_s3" + + dynamic "server_side_encryption" { + for_each = var.server_side_encryption_enabled ? [1] : [] + content { + enabled = var.server_side_encryption_enabled + key_arn = length(var.kms_key_arn) > 0 ? var.kms_key_arn : aws_kms_key.firehose_key[0].arn + key_type = "CUSTOMER_MANAGED_CMK" + } } - s3_configuration { - prefix = var.audit_prefix - role_arn = aws_iam_role.firehose.arn - bucket_arn = aws_s3_bucket.destination.arn + extended_s3_configuration { + bucket_arn = aws_s3_bucket.destination.arn + role_arn = aws_iam_role.firehose.arn + prefix = each.value.prefix + error_output_prefix = each.value.error_output_prefix + buffering_size = each.value.buffering_size + buffering_interval = each.value.buffering_interval + compression_format = each.value.compression_format } } diff --git a/terraform/addons/byo-firehose-logging-destination/target-account/iam.tf b/terraform/addons/byo-firehose-logging-destination/target-account/iam.tf index 596ef0a73f44..aed5f2036d6a 100644 --- a/terraform/addons/byo-firehose-logging-destination/target-account/iam.tf +++ b/terraform/addons/byo-firehose-logging-destination/target-account/iam.tf @@ -22,18 +22,23 @@ data "aws_iam_policy_document" "firehose" { "firehose:PutRecordBatch", ] resources = [ - aws_kinesis_firehose_delivery_stream.osquery_results.arn, - aws_kinesis_firehose_delivery_stream.osquery_status.arn + for stream in aws_kinesis_firehose_delivery_stream.fleet_log_destinations : stream.arn ] } - statement { - effect = "Allow" - actions = [ - "kms:Decrypt", - "kms:GenerateDataKey" - ] - resources = [aws_kms_key.firehose.arn] + dynamic "statement" { + for_each = var.server_side_encryption_enabled ? [1] : [] + + content { + effect = "Allow" + actions = [ + "kms:Decrypt", + "kms:GenerateDataKey", + ] + resources = [ + length(var.kms_key_arn) > 0 ? var.kms_key_arn : aws_kms_key.firehose_key[0].arn + ] + } } } @@ -45,39 +50,4 @@ resource "aws_iam_policy" "fleet_firehose" { resource "aws_iam_role_policy_attachment" "fleet_firehose" { policy_arn = aws_iam_policy.fleet_firehose.arn role = aws_iam_role.fleet_role.name -} - -data "aws_iam_policy_document" "firehose_audit" { - count = length(var.firehose_audit_name) > 0 ? 1 : 0 - statement { - effect = "Allow" - actions = [ - "firehose:DescribeDeliveryStream", - "firehose:PutRecord", - "firehose:PutRecordBatch", - ] - resources = [ - aws_kinesis_firehose_delivery_stream.fleet_audit.*.arn - ] - } - - statement { - effect = "Allow" - actions = [ - "kms:Decrypt", - "kms:GenerateDataKey" - ] - resources = [aws_kms_key.firehose.arn] - } -} - -resource "aws_iam_policy" "fleet_firehose_audit" { - count = length(var.firehose_audit_name) > 0 ? 1 : 0 - policy = data.aws_iam_policy_document.firehose_audit.*.json -} - -resource "aws_iam_role_policy_attachment" "fleet_firehose_audit" { - count = length(var.firehose_audit_name) > 0 ? 1 : 0 - policy_arn = aws_iam_policy.fleet_firehose_audit.*.arn - role = aws_iam_role.fleet_role.name } \ No newline at end of file diff --git a/terraform/addons/byo-firehose-logging-destination/target-account/outputs.tf b/terraform/addons/byo-firehose-logging-destination/target-account/outputs.tf index da4016f48839..97351d082365 100644 --- a/terraform/addons/byo-firehose-logging-destination/target-account/outputs.tf +++ b/terraform/addons/byo-firehose-logging-destination/target-account/outputs.tf @@ -6,10 +6,7 @@ output "s3_destination" { value = aws_s3_bucket.destination.arn } -output "firehose_results" { - value = aws_kinesis_firehose_delivery_stream.osquery_results.name +output "log_destinations" { + description = "Map of Firehose delivery streams' names." + value = { for key, stream in aws_kinesis_firehose_delivery_stream.fleet_log_destinations : key => stream.name } } - -output "firehose_status" { - value = aws_kinesis_firehose_delivery_stream.osquery_status.name -} \ No newline at end of file diff --git a/terraform/addons/byo-firehose-logging-destination/target-account/s3.tf b/terraform/addons/byo-firehose-logging-destination/target-account/s3.tf index 61c7692faf0b..dafe961e555e 100644 --- a/terraform/addons/byo-firehose-logging-destination/target-account/s3.tf +++ b/terraform/addons/byo-firehose-logging-destination/target-account/s3.tf @@ -1,7 +1,7 @@ data "aws_region" "current" {} data "aws_caller_identity" "current" {} data "aws_kms_alias" "s3" { - name = "aws/s3" + name = "alias/aws/s3" } resource "aws_s3_bucket" "destination" { @@ -16,11 +16,6 @@ resource "aws_s3_bucket_public_access_block" "destination" { restrict_public_buckets = true } -resource "aws_s3_bucket_acl" "destination" { - bucket = aws_s3_bucket.destination.id - acl = "private" -} - // Objects in S3 are now encrypted by default https://aws.amazon.com/blogs/aws/amazon-s3-encrypts-new-objects-by-default/ // If you need more granular control, use a customer managed KMS Key resource "aws_s3_bucket_server_side_encryption_configuration" "destination" { diff --git a/terraform/addons/byo-firehose-logging-destination/target-account/variables.tf b/terraform/addons/byo-firehose-logging-destination/target-account/variables.tf index c46beea66887..647a9dc870e7 100644 --- a/terraform/addons/byo-firehose-logging-destination/target-account/variables.tf +++ b/terraform/addons/byo-firehose-logging-destination/target-account/variables.tf @@ -3,40 +3,57 @@ variable "osquery_logging_destination_bucket_name" { description = "name of the bucket to store osquery results & status logs" } -variable "firehose_results_name" { - type = string - description = "firehose delivery stream name for osquery results logs" - default = "osquery_results" -} - -variable "firehose_status_name" { - type = string - description = "firehose delivery stream name for osquery status logs" - default = "osquery_status" -} - -variable "firehose_audit_name" { - type = string - description = "firehose delivery stream name for Fleet audit logs" - default = "" -} - variable "fleet_iam_role_arn" { type = string description = "the arn of the fleet role that firehose will assume to write data to your bucket" } -variable "results_prefix" { - default = "results/" - description = "s3 object prefix to give to results logs" +variable "log_destinations" { + description = "A map of configurations for Firehose delivery streams." + type = map(object({ + name = string + prefix = string + error_output_prefix = string + buffering_size = number + buffering_interval = number + compression_format = string + })) + default = { + results = { + name = "osquery_results" + prefix = "results/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/" + error_output_prefix = "results/error/error=!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/" + buffering_size = 20 + buffering_interval = 120 + compression_format = "UNCOMPRESSED" + }, + status = { + name = "osquery_status" + prefix = "status/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/" + error_output_prefix = "status/error/error=!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/" + buffering_size = 20 + buffering_interval = 120 + compression_format = "UNCOMPRESSED" + }, + audit = { + name = "fleet_audit" + prefix = "audit/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/" + error_output_prefix = "audit/error/error=!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/" + buffering_size = 20 + buffering_interval = 120 + compression_format = "UNCOMPRESSED" + } + } } -variable "status_prefix" { - default = "status/" - description = "s3 object prefix to give status logs" +variable "server_side_encryption_enabled" { + description = "A boolean flag to enable/disable server-side encryption. Defaults to true (enabled)." + type = bool + default = true } -variable "audit_prefix" { - default = "audit/" - description = "s3 object prefix to give Fleet audit logs" +variable "kms_key_arn" { + description = "An optional KMS key ARN for server-side encryption. If not provided and encryption is enabled, a new key will be created." + type = string + default = "" } \ No newline at end of file diff --git a/terraform/addons/byo-firehose-logging-destination/target-account/version.tf b/terraform/addons/byo-firehose-logging-destination/target-account/version.tf index 00143c571d72..529d9d07acce 100644 --- a/terraform/addons/byo-firehose-logging-destination/target-account/version.tf +++ b/terraform/addons/byo-firehose-logging-destination/target-account/version.tf @@ -4,7 +4,7 @@ terraform { required_providers { aws = { source = "hashicorp/aws" - version = ">= 4.52.0" + version = ">= 5.29.0" } } } \ No newline at end of file