Merge pull request #1247 from alphagov/samsimpson1/big-import

Import more S3 resources from govuk-aws
alphagov · Apr 17, 2024 · b5f4bbe · b5f4bbe
2 parents cf2cea0 + a3e827e
commit b5f4bbe
Show file tree

Hide file tree

Showing 6 changed files with 304 additions and 23 deletions.
diff --git a/terraform/deployments/govuk-publishing-infrastructure/asset_manager_s3.tf b/terraform/deployments/govuk-publishing-infrastructure/asset_manager_s3.tf
@@ -1,7 +1,37 @@
+resource "aws_s3_bucket" "assets" {
+  bucket = "govuk-assets-${var.govuk_environment}"
+}
+
+import {
+  to = aws_s3_bucket.assets
+  id = "govuk-assets-${var.govuk_environment}"
+}
+
+resource "aws_s3_bucket_versioning" "assets" {
+  bucket = aws_s3_bucket.assets.id
+  versioning_configuration { status = "Enabled" }
+}
+
+import {
+  to = aws_s3_bucket_versioning.assets
+  id = "govuk-assets-${var.govuk_environment}"
+}
+
+resource "aws_s3_bucket_logging" "assets" {
+  bucket        = aws_s3_bucket.assets.id
+  target_bucket = "govuk-${var.govuk_environment}-aws-logging"
+  target_prefix = "s3/govuk-assets-${var.govuk_environment}/"
+}
+
+import {
+  to = aws_s3_bucket_logging.assets
+  id = "govuk-assets-${var.govuk_environment}"
+}
+
 data "aws_iam_policy_document" "asset_manager_s3" {
   statement {
     actions   = ["s3:GetBucketLocation", "s3:ListBucket"]
-    resources = [data.terraform_remote_state.infra_assets.outputs.asset_manager_bucket_arn]
+    resources = [aws_s3_bucket.assets.arn]
   }
 
   statement {
@@ -12,7 +42,7 @@ data "aws_iam_policy_document" "asset_manager_s3" {
       "s3:*ObjectVersion",
       "s3:GetObject*Attributes"
     ]
-    resources = ["${data.terraform_remote_state.infra_assets.outputs.asset_manager_bucket_arn}/*"]
+    resources = ["${aws_s3_bucket.assets.arn}/*"]
 
   }
 }

diff --git a/terraform/deployments/govuk-publishing-infrastructure/athena_query_results_s3.tf b/terraform/deployments/govuk-publishing-infrastructure/athena_query_results_s3.tf
@@ -0,0 +1,26 @@
+resource "aws_s3_bucket" "athena_query_results" {
+  bucket = "govuk-${var.govuk_environment}-athena-query-results"
+}
+
+resource "aws_s3_bucket_lifecycle_configuration" "athena_query_results" {
+  bucket = aws_s3_bucket.athena_query_results.id
+
+  rule {
+    id     = "govuk-${var.govuk_environment}-csp-reports-lifecycle"
+    status = "Enabled"
+
+    expiration {
+      days = 7
+    }
+  }
+}
+
+import {
+  to = aws_s3_bucket.athena_query_results
+  id = "govuk-${var.govuk_environment}-athena-query-results"
+}
+
+import {
+  to = aws_s3_bucket_lifecycle_configuration.athena_query_results
+  id = "govuk-${var.govuk_environment}-athena-query-results"
+}
diff --git a/terraform/deployments/govuk-publishing-infrastructure/content_data_s3.tf b/terraform/deployments/govuk-publishing-infrastructure/content_data_s3.tf
@@ -0,0 +1,47 @@
+resource "aws_s3_bucket" "content_data_csvs" {
+  bucket = "govuk-${var.govuk_environment}-content-data-csvs"
+}
+
+resource "aws_s3_bucket_acl" "content_data_csvs" {
+  bucket = aws_s3_bucket.content_data_csvs.id
+  acl    = "public-read"
+}
+
+resource "aws_s3_bucket_logging" "content_data_csvs" {
+  bucket        = aws_s3_bucket.content_data_csvs.id
+  target_bucket = "govuk-${var.govuk_environment}-aws-logging"
+  target_prefix = "s3/govuk-${var.govuk_environment}-content-data-csvs/"
+}
+
+resource "aws_s3_bucket_lifecycle_configuration" "content_data_csvs" {
+  bucket = aws_s3_bucket.content_data_csvs.id
+
+  rule {
+    id     = "all"
+    status = "Enabled"
+
+    expiration {
+      days = 7
+    }
+  }
+}
+
+import {
+  to = aws_s3_bucket_lifecycle_configuration.content_data_csvs
+  id = "govuk-${var.govuk_environment}-content-data-csvs"
+}
+
+import {
+  to = aws_s3_bucket.content_data_csvs
+  id = "govuk-${var.govuk_environment}-content-data-csvs"
+}
+
+import {
+  to = aws_s3_bucket_logging.content_data_csvs
+  id = "govuk-${var.govuk_environment}-content-data-csvs"
+}
+
+import {
+  to = aws_s3_bucket_acl.content_data_csvs
+  id = "govuk-${var.govuk_environment}-content-data-csvs"
+}
diff --git a/terraform/deployments/govuk-publishing-infrastructure/content_publisher_activestorage_iam.tf b/terraform/deployments/govuk-publishing-infrastructure/content_publisher_activestorage_iam.tf
@@ -0,0 +1,105 @@
+# Replication IAM role/policy
+
+data "aws_iam_policy_document" "content_publisher_activestorage_replication_role" {
+  statement {
+    actions = ["sts:AssumeRole"]
+    principals {
+      type        = "Service"
+      identifiers = ["s3.amazonaws.com"]
+    }
+    effect = "Allow"
+  }
+}
+
+resource "aws_iam_role" "content_publisher_activestorage_replication_role" {
+  name               = "govuk-content-publisher-activestorage-replication-role"
+  assume_role_policy = data.aws_iam_policy_document.content_publisher_activestorage_replication_role.json
+}
+
+data "aws_iam_policy_document" "content_publisher_activestorage_replication_policy" {
+  statement {
+    actions = [
+      "s3:GetReplicationConfiguration",
+      "s3:ListBucket"
+    ]
+    resources = [aws_s3_bucket.content_publisher_activestorage.arn]
+    effect    = "Allow"
+  }
+  statement {
+    actions = [
+      "s3:GetObjectVersion",
+      "s3:GetObjectVersionAcl",
+      "s3:GetObjectVersionTagging"
+    ]
+    resources = ["${aws_s3_bucket.content_publisher_activestorage.arn}/*"]
+    effect    = "Allow"
+  }
+  statement {
+    actions = [
+      "s3:ReplicateObject",
+      "s3:ReplicateDelete"
+    ]
+    resources = ["${aws_s3_bucket.content_publisher_activestorage_replica.arn}/*"]
+  }
+}
+
+resource "aws_iam_policy" "content_publisher_activestorage_replication_policy" {
+  name        = "govuk-${var.govuk_environment}-content-publisher-activestorage-replication-policy"
+  policy      = data.aws_iam_policy_document.content_publisher_activestorage_replication_policy.json
+  description = "Allows replication of the content publisher activestorage bucket"
+}
+
+resource "aws_iam_role_policy_attachment" "content_publisher_activestorage_replication_policy" {
+  role       = aws_iam_role.content_publisher_activestorage_replication_role.name
+  policy_arn = aws_iam_policy.content_publisher_activestorage_replication_policy.arn
+}
+
+# Imports (temporary)
+
+import {
+  to = aws_iam_role.content_publisher_activestorage_replication_role
+  id = "govuk-content-publisher-activestorage-replication-role"
+}
+
+import {
+  to = aws_iam_policy.content_publisher_activestorage_replication_policy
+  id = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/govuk-${var.govuk_environment}-content-publisher-activestorage-replication-policy"
+}
+
+import {
+  to = aws_iam_role_policy_attachment.content_publisher_activestorage_replication_policy
+  id = "govuk-content-publisher-activestorage-replication-role/arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/govuk-${var.govuk_environment}-content-publisher-activestorage-replication-policy"
+}
+
+# App access role/policy
+
+data "aws_iam_policy_document" "content_publisher_s3" {
+  statement {
+    actions   = ["s3:GetBucketLocation", "s3:ListBucket"]
+    resources = [aws_s3_bucket.content_publisher_activestorage.arn]
+  }
+
+  statement {
+    actions = [
+      "s3:*MultipartUpload*",
+      "s3:*Object",
+      "s3:*ObjectAcl",
+      "s3:*ObjectVersion",
+      "s3:GetObject*Attributes"
+    ]
+    resources = ["${aws_s3_bucket.content_publisher_activestorage.arn}/*"]
+  }
+}
+
+resource "aws_iam_policy" "content_publisher_s3" {
+  name        = "content_publisher_s3"
+  description = "Read and write to this environment's content-publisher-activestorage bucket."
+
+  policy = data.aws_iam_policy_document.content_publisher_s3.json
+}
+
+# TODO: consider IRSA (pod identity) rather than granting to nodes.
+resource "aws_iam_role_policy_attachment" "content_publisher_s3" {
+  role       = data.tfe_outputs.cluster_infrastructure.nonsensitive_values.worker_iam_role_name
+  policy_arn = aws_iam_policy.content_publisher_s3.arn
+}
diff --git a/terraform/deployments/govuk-publishing-infrastructure/content_publisher_s3.tf b/terraform/deployments/govuk-publishing-infrastructure/content_publisher_s3.tf
@@ -1,30 +1,97 @@
-data "aws_iam_policy_document" "content_publisher_s3" {
-  statement {
-    actions   = ["s3:GetBucketLocation", "s3:ListBucket"]
-    resources = [data.terraform_remote_state.infra_content_publisher.outputs.activestorage_s3_bucket_arn]
+resource "aws_s3_bucket" "content_publisher_activestorage" {
+  bucket = "govuk-${var.govuk_environment}-content-publisher-activestorage"
+}
+
+resource "aws_s3_bucket_replication_configuration" "content_publisher_activestorage" {
+  bucket = aws_s3_bucket.content_publisher_activestorage.id
+  role   = aws_iam_role.content_publisher_activestorage_replication_role.arn
+
+  rule {
+    id = "govuk-content-publisher-activestorage-replication-whole-bucket-rule"
+    # Enabled in all envs except integration
+    status = var.govuk_environment == "integration" ? "Disabled" : "Enabled"
+
+    destination {
+      bucket        = aws_s3_bucket.content_publisher_activestorage_replica.arn
+      storage_class = "STANDARD"
+    }
   }
+}
+
+resource "aws_s3_bucket_logging" "content_publisher_activestorage" {
+  bucket        = aws_s3_bucket.content_publisher_activestorage.id
+  target_bucket = "govuk-${var.govuk_environment}-aws-logging"
+  target_prefix = "s3/govuk-${var.govuk_environment}-content-publisher-activestorage/"
+}
+
+resource "aws_s3_bucket_versioning" "content_publisher_activestorage" {
+  bucket = aws_s3_bucket.content_publisher_activestorage.id
+  versioning_configuration { status = "Enabled" }
+}
+
+resource "aws_s3_bucket" "content_publisher_activestorage_replica" {
+  bucket   = "govuk-${var.govuk_environment}-content-publisher-activestorage-replica"
+  provider = aws.replica
+}
+
+resource "aws_s3_bucket_versioning" "content_publisher_activestorage_replica" {
+  bucket   = aws_s3_bucket.content_publisher_activestorage_replica.id
+  provider = aws.replica
+  versioning_configuration { status = "Enabled" }
+}
 
-  statement {
-    actions = [
-      "s3:*MultipartUpload*",
-      "s3:*Object",
-      "s3:*ObjectAcl",
-      "s3:*ObjectVersion",
-      "s3:GetObject*Attributes"
-    ]
-    resources = ["${data.terraform_remote_state.infra_content_publisher.outputs.activestorage_s3_bucket_arn}/*"]
+resource "aws_s3_bucket_lifecycle_configuration" "content_publisher_activestorage_replica" {
+  bucket   = aws_s3_bucket.content_publisher_activestorage_replica.id
+  provider = aws.replica
+
+  rule {
+    id = "whole_bucket_lifecycle_rule_integration"
+    # Only enable in integration
+    status = var.govuk_environment == "integration" ? "Enabled" : "Disabled"
+
+    expiration {
+      days = 7
+    }
+
+    noncurrent_version_expiration {
+      noncurrent_days = 1
+    }
   }
 }
 
-resource "aws_iam_policy" "content_publisher_s3" {
-  name        = "content_publisher_s3"
-  description = "Read and write to this environment's content-publisher-activestorage bucket."
+# Imports (temporary)
+
+import {
+  to = aws_s3_bucket.content_publisher_activestorage
+  id = "govuk-${var.govuk_environment}-content-publisher-activestorage"
+}
+
+import {
+  to = aws_s3_bucket.content_publisher_activestorage_replica
+  id = "govuk-${var.govuk_environment}-content-publisher-activestorage-replica"
+}
+
+import {
+  to = aws_s3_bucket_versioning.content_publisher_activestorage_replica
+  id = "govuk-${var.govuk_environment}-content-publisher-activestorage-replica"
+}
+
+import {
+  to = aws_s3_bucket_lifecycle_configuration.content_publisher_activestorage_replica
+  id = "govuk-${var.govuk_environment}-content-publisher-activestorage-replica"
+}
+
+import {
+  to = aws_s3_bucket_versioning.content_publisher_activestorage
+  id = "govuk-${var.govuk_environment}-content-publisher-activestorage"
+}
 
-  policy = data.aws_iam_policy_document.content_publisher_s3.json
+import {
+  to = aws_s3_bucket_logging.content_publisher_activestorage
+  id = "govuk-${var.govuk_environment}-content-publisher-activestorage"
 }
 
-# TODO: consider IRSA (pod identity) rather than granting to nodes.
-resource "aws_iam_role_policy_attachment" "content_publisher_s3" {
-  role       = data.tfe_outputs.cluster_infrastructure.nonsensitive_values.worker_iam_role_name
-  policy_arn = aws_iam_policy.content_publisher_s3.arn
+import {
+  to = aws_s3_bucket_replication_configuration.content_publisher_activestorage
+  id = "govuk-${var.govuk_environment}-content-publisher-activestorage"
 }
diff --git a/terraform/deployments/govuk-publishing-infrastructure/main.tf b/terraform/deployments/govuk-publishing-infrastructure/main.tf
@@ -50,6 +50,12 @@ provider "aws" {
   default_tags { tags = local.default_tags }
 }
 
+provider "aws" {
+  region = "eu-west-2"
+  alias  = "replica"
+  default_tags { tags = local.default_tags }
+}
+
 provider "random" {}
 
 # used by the fastly ip ranges provider.