From 43f23042370bc15cf4c05d928d1de52c58df17ac Mon Sep 17 00:00:00 2001 From: Aaron Robinson Date: Thu, 12 Sep 2024 16:07:54 +0100 Subject: [PATCH 1/6] s3 bucket and iam roles for grafana-athena --- terraform/environments/core-logging/iam.tf | 36 +++++++++++++ .../environments/core-logging/s3_logging.tf | 54 +++++++++++++++++++ 2 files changed, 90 insertions(+) diff --git a/terraform/environments/core-logging/iam.tf b/terraform/environments/core-logging/iam.tf index 096ad138f..b2bb3cabd 100644 --- a/terraform/environments/core-logging/iam.tf +++ b/terraform/environments/core-logging/iam.tf @@ -53,3 +53,39 @@ resource "aws_iam_role_policy_attachment" "vpc_flow_log_publish_policy" { role = aws_iam_role.vpc_flow_log.id policy_arn = aws_iam_policy.vpc_flow_log_publish_policy.arn } + +# Grafana-Athena Role +resource "aws_iam_role" "grafana_athena" { + name = "grafana-athena" + assume_role_policy = data.aws_iam_policy_document.grafana-athena.json +} + +# Grafana-Athena Policy +data "aws_iam_policy_document" "grafana_athena_policy" { + statement { + sid = "s3Access" + effect = "Allow" + + actions = [ + "s3:GetObject", + "s3:PutObject", + "s3:ListBucket" + ] + + resources = [ + module.s3-grafana-athena-query-results.bucket.arn, + "${module.s3-grafana-athena-query-results.bucket.arn}/*" + ] + + principals { + type = "AWS" + identifiers = [aws_iam_role.ssm_role.arn] + } + } +} + +# Attach AmazonGrafanaAthenaAccess policy +resource "aws_iam_role_policy_attachment" "grafana_athena_attachment" { + role = aws_iam_role.grafana_athena.name + policy_arn = "arn:aws:iam::aws:policy/AmazonGrafanaAthenaAccess" +} diff --git a/terraform/environments/core-logging/s3_logging.tf b/terraform/environments/core-logging/s3_logging.tf index f049c6724..6d5feed83 100644 --- a/terraform/environments/core-logging/s3_logging.tf +++ b/terraform/environments/core-logging/s3_logging.tf @@ -346,3 +346,57 @@ module "s3-bucket-cloudtrail-logging" { tags = local.tags } + +module "s3-grafana-athena-query-results" { + source = "github.com/ministryofjustice/modernisation-platform-terraform-s3-bucket?ref=4e17731f72ef24b804207f55b182f49057e73ec9" # v8.1.0 + bucket_prefix = "grafana-athena-query-results-" + versioning_enabled = true + ownership_controls = "BucketOwnerEnforced" + replication_enabled = false + providers = { + aws.bucket-replication = aws + } + + lifecycle_rule = [ + { + id = "main" + enabled = "Enabled" + prefix = "" + + tags = { + rule = "log" + autoclean = "true" + } + + transition = [ + { + days = 90 + storage_class = "STANDARD_IA" + }, { + days = 365 + storage_class = "GLACIER" + } + ] + + expiration = { + days = 730 + } + + noncurrent_version_transition = [ + { + days = 90 + storage_class = "STANDARD_IA" + }, { + days = 365 + storage_class = "GLACIER" + } + ] + + noncurrent_version_expiration = { + days = 730 + } + } + ] + + tags = local.tags +} From c108fa69c99aa8887c8374fdbbbff7f32be51c33 Mon Sep 17 00:00:00 2001 From: Aaron Robinson Date: Thu, 12 Sep 2024 16:15:56 +0100 Subject: [PATCH 2/6] fixup typo --- terraform/environments/core-logging/iam.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/environments/core-logging/iam.tf b/terraform/environments/core-logging/iam.tf index b2bb3cabd..64253e519 100644 --- a/terraform/environments/core-logging/iam.tf +++ b/terraform/environments/core-logging/iam.tf @@ -57,7 +57,7 @@ resource "aws_iam_role_policy_attachment" "vpc_flow_log_publish_policy" { # Grafana-Athena Role resource "aws_iam_role" "grafana_athena" { name = "grafana-athena" - assume_role_policy = data.aws_iam_policy_document.grafana-athena.json + assume_role_policy = data.aws_iam_policy_document.grafana_athena.json } # Grafana-Athena Policy @@ -79,7 +79,7 @@ data "aws_iam_policy_document" "grafana_athena_policy" { principals { type = "AWS" - identifiers = [aws_iam_role.ssm_role.arn] + identifiers = [aws_iam_role.grafana_athena.arn] } } } From 4e040b1508ee67288f7be18e8d02b5287cbee4d7 Mon Sep 17 00:00:00 2001 From: Aaron Robinson Date: Thu, 12 Sep 2024 16:20:48 +0100 Subject: [PATCH 3/6] fix policy name --- terraform/environments/core-logging/iam.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/environments/core-logging/iam.tf b/terraform/environments/core-logging/iam.tf index 64253e519..fd6f064d3 100644 --- a/terraform/environments/core-logging/iam.tf +++ b/terraform/environments/core-logging/iam.tf @@ -57,7 +57,7 @@ resource "aws_iam_role_policy_attachment" "vpc_flow_log_publish_policy" { # Grafana-Athena Role resource "aws_iam_role" "grafana_athena" { name = "grafana-athena" - assume_role_policy = data.aws_iam_policy_document.grafana_athena.json + assume_role_policy = data.aws_iam_policy_document.grafana_athena_policy.json } # Grafana-Athena Policy From da963eee7e482f4bfc449788cc746d15f16b8570 Mon Sep 17 00:00:00 2001 From: Aaron Robinson Date: Thu, 12 Sep 2024 16:45:40 +0100 Subject: [PATCH 4/6] update tags --- terraform/environments/core-logging/s3_logging.tf | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/terraform/environments/core-logging/s3_logging.tf b/terraform/environments/core-logging/s3_logging.tf index 6d5feed83..2fa4cdf91 100644 --- a/terraform/environments/core-logging/s3_logging.tf +++ b/terraform/environments/core-logging/s3_logging.tf @@ -398,5 +398,7 @@ module "s3-grafana-athena-query-results" { } ] - tags = local.tags + tags = merge(local.tags, + { Name = lower(format("s3-bucket-%s-%s-grafana-athena", local.application_name, local.environment)) } + ) } From 2411727290e2cf90912b6819a144bf1dfdb8eba8 Mon Sep 17 00:00:00 2001 From: Aaron Robinson Date: Fri, 13 Sep 2024 08:56:55 +0100 Subject: [PATCH 5/6] move to observability.tf --- terraform/environments/core-logging/iam.tf | 36 ------------ .../core-logging/observability.tf | 56 +++++++++++++++++++ .../environments/core-logging/s3_logging.tf | 56 ------------------- 3 files changed, 56 insertions(+), 92 deletions(-) diff --git a/terraform/environments/core-logging/iam.tf b/terraform/environments/core-logging/iam.tf index fd6f064d3..096ad138f 100644 --- a/terraform/environments/core-logging/iam.tf +++ b/terraform/environments/core-logging/iam.tf @@ -53,39 +53,3 @@ resource "aws_iam_role_policy_attachment" "vpc_flow_log_publish_policy" { role = aws_iam_role.vpc_flow_log.id policy_arn = aws_iam_policy.vpc_flow_log_publish_policy.arn } - -# Grafana-Athena Role -resource "aws_iam_role" "grafana_athena" { - name = "grafana-athena" - assume_role_policy = data.aws_iam_policy_document.grafana_athena_policy.json -} - -# Grafana-Athena Policy -data "aws_iam_policy_document" "grafana_athena_policy" { - statement { - sid = "s3Access" - effect = "Allow" - - actions = [ - "s3:GetObject", - "s3:PutObject", - "s3:ListBucket" - ] - - resources = [ - module.s3-grafana-athena-query-results.bucket.arn, - "${module.s3-grafana-athena-query-results.bucket.arn}/*" - ] - - principals { - type = "AWS" - identifiers = [aws_iam_role.grafana_athena.arn] - } - } -} - -# Attach AmazonGrafanaAthenaAccess policy -resource "aws_iam_role_policy_attachment" "grafana_athena_attachment" { - role = aws_iam_role.grafana_athena.name - policy_arn = "arn:aws:iam::aws:policy/AmazonGrafanaAthenaAccess" -} diff --git a/terraform/environments/core-logging/observability.tf b/terraform/environments/core-logging/observability.tf index 1d21e906d..d832f8251 100644 --- a/terraform/environments/core-logging/observability.tf +++ b/terraform/environments/core-logging/observability.tf @@ -6,3 +6,59 @@ module "observability_platform_tenant" { tags = local.tags } + +module "s3-grafana-athena-query-results" { + source = "github.com/ministryofjustice/modernisation-platform-terraform-s3-bucket?ref=4e17731f72ef24b804207f55b182f49057e73ec9" # v8.1.0 + bucket_prefix = "grafana-athena-query-results-" + versioning_enabled = true + ownership_controls = "BucketOwnerEnforced" + replication_enabled = false + providers = { + aws.bucket-replication = aws + } + + lifecycle_rule = [ + { + id = "main" + enabled = "Enabled" + prefix = "" + + tags = { + rule = "log" + autoclean = "true" + } + + transition = [ + { + days = 90 + storage_class = "STANDARD_IA" + }, { + days = 365 + storage_class = "GLACIER" + } + ] + + expiration = { + days = 730 + } + + noncurrent_version_transition = [ + { + days = 90 + storage_class = "STANDARD_IA" + }, { + days = 365 + storage_class = "GLACIER" + } + ] + + noncurrent_version_expiration = { + days = 730 + } + } + ] + + tags = merge(local.tags, + { Name = lower(format("s3-bucket-%s-%s-grafana-athena", local.application_name, local.environment)) } + ) +} diff --git a/terraform/environments/core-logging/s3_logging.tf b/terraform/environments/core-logging/s3_logging.tf index 2fa4cdf91..f049c6724 100644 --- a/terraform/environments/core-logging/s3_logging.tf +++ b/terraform/environments/core-logging/s3_logging.tf @@ -346,59 +346,3 @@ module "s3-bucket-cloudtrail-logging" { tags = local.tags } - -module "s3-grafana-athena-query-results" { - source = "github.com/ministryofjustice/modernisation-platform-terraform-s3-bucket?ref=4e17731f72ef24b804207f55b182f49057e73ec9" # v8.1.0 - bucket_prefix = "grafana-athena-query-results-" - versioning_enabled = true - ownership_controls = "BucketOwnerEnforced" - replication_enabled = false - providers = { - aws.bucket-replication = aws - } - - lifecycle_rule = [ - { - id = "main" - enabled = "Enabled" - prefix = "" - - tags = { - rule = "log" - autoclean = "true" - } - - transition = [ - { - days = 90 - storage_class = "STANDARD_IA" - }, { - days = 365 - storage_class = "GLACIER" - } - ] - - expiration = { - days = 730 - } - - noncurrent_version_transition = [ - { - days = 90 - storage_class = "STANDARD_IA" - }, { - days = 365 - storage_class = "GLACIER" - } - ] - - noncurrent_version_expiration = { - days = 730 - } - } - ] - - tags = merge(local.tags, - { Name = lower(format("s3-bucket-%s-%s-grafana-athena", local.application_name, local.environment)) } - ) -} From 709d366dc67400f5a9c0e9c4a14d2bb7fc5cddc3 Mon Sep 17 00:00:00 2001 From: Aaron Robinson Date: Fri, 13 Sep 2024 09:09:28 +0100 Subject: [PATCH 6/6] fix cycle dependency --- .../core-logging/observability.tf | 55 +++++++++++++++++-- 1 file changed, 51 insertions(+), 4 deletions(-) diff --git a/terraform/environments/core-logging/observability.tf b/terraform/environments/core-logging/observability.tf index d832f8251..c7ac96d51 100644 --- a/terraform/environments/core-logging/observability.tf +++ b/terraform/environments/core-logging/observability.tf @@ -6,9 +6,58 @@ module "observability_platform_tenant" { tags = local.tags } +# Grafana-Athena Role +resource "aws_iam_role" "grafana_athena" { + name = "grafana-athena" + assume_role_policy = data.aws_iam_policy_document.grafana_athena_assume_role_policy.json +} + +# Assume Role Policy for Grafana-Athena +data "aws_iam_policy_document" "grafana_athena_assume_role_policy" { + statement { + actions = ["sts:AssumeRole"] + + principals { + type = "Service" + identifiers = ["athena.amazonaws.com"] + } + } +} + +# Grafana-Athena S3 Access Policy (Note: remove aws_iam_role reference) +data "aws_iam_policy_document" "grafana_athena_policy" { + statement { + sid = "s3Access" + effect = "Allow" + + actions = [ + "s3:GetObject", + "s3:PutObject", + "s3:ListBucket" + ] + resources = [ + module.s3-grafana-athena-query-results.bucket.arn, + "${module.s3-grafana-athena-query-results.bucket.arn}/*" + ] + + principals { + type = "AWS" + # Use a placeholder ARN for the role to avoid circular dependency + identifiers = [data.aws_caller_identity.current.account_id] + } + } +} + +# Attach AmazonGrafanaAthenaAccess policy +resource "aws_iam_role_policy_attachment" "grafana_athena_attachment" { + role = aws_iam_role.grafana_athena.id + policy_arn = "arn:aws:iam::aws:policy/AmazonGrafanaAthenaAccess" +} + +# S3 bucket for Grafana Athena query results module "s3-grafana-athena-query-results" { - source = "github.com/ministryofjustice/modernisation-platform-terraform-s3-bucket?ref=4e17731f72ef24b804207f55b182f49057e73ec9" # v8.1.0 + source = "github.com/ministryofjustice/modernisation-platform-terraform-s3-bucket?ref=4e17731f72ef24b804207f55b182f49057e73ec9" bucket_prefix = "grafana-athena-query-results-" versioning_enabled = true ownership_controls = "BucketOwnerEnforced" @@ -58,7 +107,5 @@ module "s3-grafana-athena-query-results" { } ] - tags = merge(local.tags, - { Name = lower(format("s3-bucket-%s-%s-grafana-athena", local.application_name, local.environment)) } - ) + tags = local.tags }