Skip to content

Commit

Permalink
refactor module to enable opt-out of firehose encryption-at-rest
Browse files Browse the repository at this point in the history
  • Loading branch information
@edwardsb
edwardsb committed Dec 5, 2023
1 parent 8cba28b commit cfcc69f
Show file tree
Hide file tree
Showing 5 changed files with 91 additions and 200 deletions.
31 changes: 7 additions & 24 deletions terraform/addons/byo-firehose-logging-destination/target-account/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ This code sets up a secure and controlled environment for the Fleet application

| Name | Version |
|------|---------|
| <a name="provider_aws"></a> [aws](#provider\_aws) | 5.29.0 |
| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 5.29.0 |

## Modules

Expand All @@ -35,10 +35,8 @@ No modules.
| [aws_iam_role.fleet_role](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
| [aws_iam_role_policy_attachment.firehose](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
| [aws_iam_role_policy_attachment.fleet_firehose](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
| [aws_kinesis_firehose_delivery_stream.fleet_audit](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kinesis_firehose_delivery_stream) | resource |
| [aws_kinesis_firehose_delivery_stream.osquery_results](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kinesis_firehose_delivery_stream) | resource |
| [aws_kinesis_firehose_delivery_stream.osquery_status](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kinesis_firehose_delivery_stream) | resource |
| [aws_kms_key.firehose](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource |
| [aws_kinesis_firehose_delivery_stream.fleet_log_destinations](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kinesis_firehose_delivery_stream) | resource |
| [aws_kms_key.firehose_key](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/kms_key) | resource |
| [aws_s3_bucket.destination](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket) | resource |
| [aws_s3_bucket_public_access_block.destination](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_public_access_block) | resource |
| [aws_s3_bucket_server_side_encryption_configuration.destination](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/s3_bucket_server_side_encryption_configuration) | resource |
Expand All @@ -54,31 +52,16 @@ No modules.

| Name | Description | Type | Default | Required |
|------|-------------|------|---------|:--------:|
| <a name="input_audit_buffering_interval"></a> [audit\_buffering\_interval](#input\_audit\_buffering\_interval) | size of the time buffer in seconds before messages are flushed to S3 | `number` | `120` | no |
| <a name="input_audit_buffering_size"></a> [audit\_buffering\_size](#input\_audit\_buffering\_size) | size of the buffer in megabytes before messages are flushed to S3 | `number` | `20` | no |
| <a name="input_audit_error_prefix"></a> [audit\_error\_prefix](#input\_audit\_error\_prefix) | s3 object prefix to give firehose audit error logs | `string` | `"audit/error/error=!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"` | no |
| <a name="input_audit_prefix"></a> [audit\_prefix](#input\_audit\_prefix) | s3 object prefix to give Fleet audit logs | `string` | `"audit/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"` | no |
| <a name="input_firehose_audit_name"></a> [firehose\_audit\_name](#input\_firehose\_audit\_name) | firehose delivery stream name for Fleet audit logs | `string` | `"fleet_audit"` | no |
| <a name="input_firehose_results_name"></a> [firehose\_results\_name](#input\_firehose\_results\_name) | firehose delivery stream name for osquery results logs | `string` | `"osquery_results"` | no |
| <a name="input_firehose_status_name"></a> [firehose\_status\_name](#input\_firehose\_status\_name) | firehose delivery stream name for osquery status logs | `string` | `"osquery_status"` | no |
| <a name="input_fleet_iam_role_arn"></a> [fleet\_iam\_role\_arn](#input\_fleet\_iam\_role\_arn) | the arn of the fleet role that firehose will assume to write data to your bucket | `string` | n/a | yes |
| <a name="input_kms_key_arn"></a> [kms\_key\_arn](#input\_kms\_key\_arn) | An optional KMS key ARN for server-side encryption. If not provided and encryption is enabled, a new key will be created. | `string` | `""` | no |
| <a name="input_log_destinations"></a> [log\_destinations](#input\_log\_destinations) | A map of configurations for Firehose delivery streams. | <pre>map(object({<br> name = string<br> prefix = string<br> error_output_prefix = string<br> buffering_size = number<br> buffering_interval = number<br> compression_format = string<br> }))</pre> | <pre>{<br> "audit": {<br> "buffering_interval": 120,<br> "buffering_size": 20,<br> "compression_format": "UNCOMPRESSED",<br> "error_output_prefix": "audit/error/error=!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/",<br> "name": "fleet_audit",<br> "prefix": "audit/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"<br> },<br> "results": {<br> "buffering_interval": 120,<br> "buffering_size": 20,<br> "compression_format": "UNCOMPRESSED",<br> "error_output_prefix": "results/error/error=!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/",<br> "name": "osquery_results",<br> "prefix": "results/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"<br> },<br> "status": {<br> "buffering_interval": 120,<br> "buffering_size": 20,<br> "compression_format": "UNCOMPRESSED",<br> "error_output_prefix": "status/error/error=!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/",<br> "name": "osquery_status",<br> "prefix": "status/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"<br> }<br>}</pre> | no |
| <a name="input_osquery_logging_destination_bucket_name"></a> [osquery\_logging\_destination\_bucket\_name](#input\_osquery\_logging\_destination\_bucket\_name) | name of the bucket to store osquery results & status logs | `string` | n/a | yes |
| <a name="input_results_buffering_interval"></a> [results\_buffering\_interval](#input\_results\_buffering\_interval) | size of the time buffer in seconds before messages are flushed to S3 | `number` | `120` | no |
| <a name="input_results_buffering_size"></a> [results\_buffering\_size](#input\_results\_buffering\_size) | size of the buffer in megabytes before messages are flushed to S3 | `number` | `20` | no |
| <a name="input_results_compression_format"></a> [results\_compression\_format](#input\_results\_compression\_format) | n/a | `string` | `"UNCOMPRESSED"` | no |
| <a name="input_results_error_prefix"></a> [results\_error\_prefix](#input\_results\_error\_prefix) | s3 object prefix to give firehose results error logs | `string` | `"results/error/error=!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"` | no |
| <a name="input_results_prefix"></a> [results\_prefix](#input\_results\_prefix) | s3 object prefix to give to results logs | `string` | `"results/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"` | no |
| <a name="input_status_buffering_interval"></a> [status\_buffering\_interval](#input\_status\_buffering\_interval) | size of the time buffer in seconds before messages are flushed to S3 | `number` | `120` | no |
| <a name="input_status_buffering_size"></a> [status\_buffering\_size](#input\_status\_buffering\_size) | size of the buffer in megabytes before messages are flushed to S3 | `number` | `20` | no |
| <a name="input_status_error_prefix"></a> [status\_error\_prefix](#input\_status\_error\_prefix) | s3 object prefix to give firehose status error logs | `string` | `"status/error/error=!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"` | no |
| <a name="input_status_prefix"></a> [status\_prefix](#input\_status\_prefix) | s3 object prefix to give status logs | `string` | `"status/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"` | no |
| <a name="input_server_side_encryption_enabled"></a> [server\_side\_encryption\_enabled](#input\_server\_side\_encryption\_enabled) | A boolean flag to enable/disable server-side encryption. Defaults to true (enabled). | `bool` | `true` | no |

## Outputs

| Name | Description |
|------|-------------|
| <a name="output_firehose_audit"></a> [firehose\_audit](#output\_firehose\_audit) | n/a |
| <a name="output_firehose_iam_role"></a> [firehose\_iam\_role](#output\_firehose\_iam\_role) | n/a |
| <a name="output_firehose_results"></a> [firehose\_results](#output\_firehose\_results) | n/a |
| <a name="output_firehose_status"></a> [firehose\_status](#output\_firehose\_status) | n/a |
| <a name="output_log_destinations"></a> [log\_destinations](#output\_log\_destinations) | Map of Firehose delivery streams' names. |
| <a name="output_s3_destination"></a> [s3\_destination](#output\_s3\_destination) | n/a |
78 changes: 19 additions & 59 deletions terraform/addons/byo-firehose-logging-destination/target-account/firehose.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,9 +31,7 @@ data "aws_iam_policy_document" "firehose_policy" {
effect = "Allow"
actions = ["logs:PutLogEvents"]
resources = [
"arn:aws:logs:${data.aws_region.current.id}:${data.aws_caller_identity.current.account_id}:log-group:/aws/kinesisfirehose/${var.firehose_results_name}:*",
"arn:aws:logs:${data.aws_region.current.id}:${data.aws_caller_identity.current.account_id}:log-group:/aws/kinesisfirehose/${var.firehose_status_name}:*",
"arn:aws:logs:${data.aws_region.current.id}:${data.aws_caller_identity.current.account_id}:log-group:/aws/kinesisfirehose/${var.firehose_audit_name}:*"
for name in keys(var.log_destinations) : "arn:aws:logs:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:log-group:/aws/kinesisfirehose/${var.log_destinations[name].name}:*"
]
}

Expand Down Expand Up @@ -61,70 +59,32 @@ resource "aws_iam_role_policy_attachment" "firehose" {
role = aws_iam_role.firehose.name
}

resource "aws_kms_key" "firehose" {
enable_key_rotation = true
description = "key to encrypt firehose in-transit data"
resource "aws_kms_key" "firehose_key" {
count = var.server_side_encryption_enabled && length(var.kms_key_arn) == 0 ? 1 : 0
description = "KMS key for encrypting Firehose data."
}

resource "aws_kinesis_firehose_delivery_stream" "osquery_results" {
name = var.firehose_results_name
resource "aws_kinesis_firehose_delivery_stream" "fleet_log_destinations" {
for_each = var.log_destinations
name = each.value.name
destination = "extended_s3"

server_side_encryption {
enabled = true
key_arn = aws_kms_key.firehose.arn
key_type = "CUSTOMER_MANAGED_CMK"
}

extended_s3_configuration {
bucket_arn = aws_s3_bucket.destination.arn
role_arn = aws_iam_role.firehose.arn
prefix = var.results_prefix
error_output_prefix = var.results_error_prefix
buffering_size = var.results_buffering_size
buffering_interval = var.results_buffering_interval
compression_format = var.results_compression_format
}
}

resource "aws_kinesis_firehose_delivery_stream" "osquery_status" {
name = var.firehose_status_name
destination = "extended_s3"

server_side_encryption {
enabled = true
key_arn = aws_kms_key.firehose.arn
key_type = "CUSTOMER_MANAGED_CMK"
}

extended_s3_configuration {
bucket_arn = aws_s3_bucket.destination.arn
role_arn = aws_iam_role.firehose.arn
prefix = var.status_prefix
error_output_prefix = var.status_error_prefix
buffering_size = var.status_buffering_size
buffering_interval = var.status_buffering_interval
compression_format = var.status_compression_format
}
}

resource "aws_kinesis_firehose_delivery_stream" "fleet_audit" {
name = var.firehose_audit_name
destination = "extended_s3"

server_side_encryption {
enabled = true
key_arn = aws_kms_key.firehose.arn
key_type = "CUSTOMER_MANAGED_CMK"
dynamic "server_side_encryption" {
for_each = var.server_side_encryption_enabled ? [1] : []
content {
enabled = var.server_side_encryption_enabled
key_arn = length(var.kms_key_arn) > 0 ? var.kms_key_arn : aws_kms_key.firehose_key[0].arn
key_type = "CUSTOMER_MANAGED_CMK"
}
}

extended_s3_configuration {
bucket_arn = aws_s3_bucket.destination.arn
role_arn = aws_iam_role.firehose.arn
prefix = var.audit_prefix
error_output_prefix = var.audit_error_prefix
buffering_size = var.audit_buffering_size
buffering_interval = var.audit_buffering_interval
compression_format = var.audit_compression_format
prefix = each.value.prefix
error_output_prefix = each.value.error_output_prefix
buffering_size = each.value.buffering_size
buffering_interval = each.value.buffering_interval
compression_format = each.value.compression_format
}
}
24 changes: 14 additions & 10 deletions terraform/addons/byo-firehose-logging-destination/target-account/iam.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,19 +22,23 @@ data "aws_iam_policy_document" "firehose" {
"firehose:PutRecordBatch",
]
resources = [
aws_kinesis_firehose_delivery_stream.osquery_results.arn,
aws_kinesis_firehose_delivery_stream.osquery_status.arn,
aws_kinesis_firehose_delivery_stream.fleet_audit.arn
for stream in aws_kinesis_firehose_delivery_stream.fleet_log_destinations : stream.arn
]
}

statement {
effect = "Allow"
actions = [
"kms:Decrypt",
"kms:GenerateDataKey"
]
resources = [aws_kms_key.firehose.arn]
dynamic "statement" {
for_each = var.server_side_encryption_enabled ? [1] : []

content {
effect = "Allow"
actions = [
"kms:Decrypt",
"kms:GenerateDataKey",
]
resources = [
length(var.kms_key_arn) > 0 ? var.kms_key_arn : aws_kms_key.firehose_key[0].arn
]
}
}

}
Expand Down
13 changes: 3 additions & 10 deletions terraform/addons/byo-firehose-logging-destination/target-account/outputs.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,14 +6,7 @@ output "s3_destination" {
value = aws_s3_bucket.destination.arn
}

output "firehose_results" {
value = aws_kinesis_firehose_delivery_stream.osquery_results.name
output "log_destinations" {
description = "Map of Firehose delivery streams' names."
value = { for key, stream in aws_kinesis_firehose_delivery_stream.fleet_log_destinations : key => stream.name }
}

output "firehose_status" {
value = aws_kinesis_firehose_delivery_stream.osquery_status.name
}

output "firehose_audit" {
value = aws_kinesis_firehose_delivery_stream.fleet_audit.name
}
145 changes: 48 additions & 97 deletions terraform/addons/byo-firehose-logging-destination/target-account/variables.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,106 +3,57 @@ variable "osquery_logging_destination_bucket_name" {
description = "name of the bucket to store osquery results & status logs"
}

variable "firehose_results_name" {
type = string
description = "firehose delivery stream name for osquery results logs"
default = "osquery_results"
}

variable "firehose_status_name" {
type = string
description = "firehose delivery stream name for osquery status logs"
default = "osquery_status"
}

variable "firehose_audit_name" {
type = string
description = "firehose delivery stream name for Fleet audit logs"
default = "fleet_audit"
}

variable "fleet_iam_role_arn" {
type = string
description = "the arn of the fleet role that firehose will assume to write data to your bucket"
}

variable "results_prefix" {
default = "results/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"
description = "s3 object prefix to give to results logs"
}

variable "results_error_prefix" {
default = "results/error/error=!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"
description = "s3 object prefix to give firehose results error logs"
}

variable "status_prefix" {
default = "status/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"
description = "s3 object prefix to give status logs"
}

variable "status_error_prefix" {
default = "status/error/error=!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"
description = "s3 object prefix to give firehose status error logs"
}

variable "audit_prefix" {
default = "audit/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"
description = "s3 object prefix to give Fleet audit logs"
}

variable "audit_error_prefix" {
default = "audit/error/error=!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"
description = "s3 object prefix to give firehose audit error logs"
}

variable "results_buffering_size" {
type = number
default = 20
description = "size of the buffer in megabytes before messages are flushed to S3"
}

variable "results_buffering_interval" {
type = number
default = 120
description = "size of the time buffer in seconds before messages are flushed to S3"
}

variable "results_compression_format" {
type = string
default = "UNCOMPRESSED"
}

variable "status_buffering_size" {
type = number
default = 20
description = "size of the buffer in megabytes before messages are flushed to S3"
}

variable "status_buffering_interval" {
type = number
default = 120
description = "size of the time buffer in seconds before messages are flushed to S3"
}

variable "status_compression_format" {
type = string
default = "UNCOMPRESSED"
}

variable "audit_buffering_size" {
type = number
default = 20
description = "size of the buffer in megabytes before messages are flushed to S3"
}

variable "audit_buffering_interval" {
type = number
default = 120
description = "size of the time buffer in seconds before messages are flushed to S3"
}

variable "audit_compression_format" {
type = string
default = "UNCOMPRESSED"
variable "log_destinations" {
description = "A map of configurations for Firehose delivery streams."
type = map(object({
name = string
prefix = string
error_output_prefix = string
buffering_size = number
buffering_interval = number
compression_format = string
}))
default = {
results = {
name = "osquery_results"
prefix = "results/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"
error_output_prefix = "results/error/error=!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"
buffering_size = 20
buffering_interval = 120
compression_format = "UNCOMPRESSED"
},
status = {
name = "osquery_status"
prefix = "status/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"
error_output_prefix = "status/error/error=!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"
buffering_size = 20
buffering_interval = 120
compression_format = "UNCOMPRESSED"
},
audit = {
name = "fleet_audit"
prefix = "audit/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"
error_output_prefix = "audit/error/error=!{firehose:error-output-type}/year=!{timestamp:yyyy}/month=!{timestamp:MM}/day=!{timestamp:dd}/"
buffering_size = 20
buffering_interval = 120
compression_format = "UNCOMPRESSED"
}
}
}

variable "server_side_encryption_enabled" {
description = "A boolean flag to enable/disable server-side encryption. Defaults to true (enabled)."
type = bool
default = true
}

variable "kms_key_arn" {
description = "An optional KMS key ARN for server-side encryption. If not provided and encryption is enabled, a new key will be created."
type = string
default = ""
}

0 comments on commit cfcc69f

Please sign in to comment.