Skip to content

Commit

Permalink
Merge pull request #1275 from alphagov/samsimpson1/big-import
Browse files Browse the repository at this point in the history 
Import database backup buckets from govuk-aws and small GCP changes
  • Loading branch information
@samsimpson1
samsimpson1 authored Apr 25, 2024
2 parents 293285f + 4f451dd commit c316686
Show file tree
Hide file tree
Showing 8 changed files with 341 additions and 39 deletions.
54 changes: 54 additions & 0 deletions terraform/deployments/govuk-publishing-infrastructure/db_backup_iam.tf
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
locals {
db_backup_service_account_name = "db-backup"
env_also_reads_from = {
production = "govuk-production-database-backups-replica"
staging = "govuk-production-database-backups"
integration = "govuk-staging-database-backups"
}
}

module "db_backup_iam_role" {
source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
version = "~> 5.20"

role_name = "${local.db_backup_service_account_name}-${data.tfe_outputs.cluster_infrastructure.nonsensitive_values.cluster_id}"
role_description = "Role for database backup jobs. Corresponds to ${local.db_backup_service_account_name} k8s ServiceAccount."
max_session_duration = 28800

role_policy_arns = { policy = aws_iam_policy.db_backup_s3.arn }
oidc_providers = {
main = {
provider_arn = data.tfe_outputs.cluster_infrastructure.nonsensitive_values.cluster_oidc_provider_arn
namespace_service_accounts = ["apps:${local.db_backup_service_account_name}"]
}
}
}

data "aws_iam_policy_document" "db_backup_s3" {
statement {
sid = "Read"
actions = [
"s3:GetBucketLocation",
"s3:ListBucket",
"s3:GetObject",
"s3:GetObject*Attributes",
]
resources = [
"arn:aws:s3:::govuk-${var.govuk_environment}-database-backups",
"arn:aws:s3:::govuk-${var.govuk_environment}-database-backups/*",
"arn:aws:s3:::${local.env_also_reads_from[var.govuk_environment]}",
"arn:aws:s3:::${local.env_also_reads_from[var.govuk_environment]}/*",
]
}
statement {
sid = "Write"
actions = ["s3:*MultipartUpload*", "s3:PutObject"]
resources = ["arn:aws:s3:::govuk-${var.govuk_environment}-database-backups/*"]
}
}

resource "aws_iam_policy" "db_backup_s3" {
name = "db_backup_s3"
description = "Permissions over this environment's govuk-*-database-backups bucket."
policy = data.aws_iam_policy_document.db_backup_s3.json
}
212 changes: 178 additions & 34 deletions terraform/deployments/govuk-publishing-infrastructure/db_backup_s3.tf
View file
Original file line number Diff line number Diff line change
@@ -1,54 +1,198 @@
locals {
db_backup_service_account_name = "db-backup"
env_also_reads_from = {
"production" = "govuk-production-database-backups-replica"
"staging" = "govuk-production-database-backups"
"integration" = "govuk-staging-database-backups"
timelock_enabled = var.govuk_environment == "production"
timelock_days = 120
}

resource "aws_s3_bucket" "backup_main" {
bucket = "govuk-${var.govuk_environment}-database-backups"
object_lock_enabled = local.timelock_enabled
tags = { Name = "govuk-${var.govuk_environment}-database-backups" }
}

resource "aws_s3_bucket" "backup_replica" {
bucket = "govuk-${var.govuk_environment}-database-backups-replica"
provider = aws.replica
object_lock_enabled = local.timelock_enabled
tags = { Name = "govuk-${var.govuk_environment}-database-backups-replica" }
}

resource "aws_s3_bucket_object_lock_configuration" "backup_main" {
count = local.timelock_enabled ? 1 : 0

bucket = aws_s3_bucket.backup_main.id
rule {
default_retention {
mode = "COMPLIANCE"
days = local.timelock_days
}
}
}

resource "aws_s3_bucket_object_lock_configuration" "backup_replica" {
count = local.timelock_enabled ? 1 : 0

bucket = aws_s3_bucket.backup_replica.id
provider = aws.replica
rule {
default_retention {
mode = "COMPLIANCE"
days = local.timelock_days
}
}
}

module "db_backup_iam_role" {
source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
version = "~> 5.20"
resource "aws_s3_bucket_public_access_block" "backup_main" {
bucket = aws_s3_bucket.backup_main.id
block_public_acls = true
block_public_policy = true
ignore_public_acls = true
restrict_public_buckets = true
}

resource "aws_s3_bucket_public_access_block" "backup_replica" {
bucket = aws_s3_bucket.backup_replica.id
provider = aws.replica
block_public_acls = true
block_public_policy = true
ignore_public_acls = true
restrict_public_buckets = true
}

resource "aws_s3_bucket_logging" "backup_main" {
bucket = aws_s3_bucket.backup_main.id
target_bucket = "govuk-${var.govuk_environment}-aws-logging"
target_prefix = "s3/govuk-${var.govuk_environment}-database-backups/"
}

resource "aws_s3_bucket_logging" "backup_replica" {
bucket = aws_s3_bucket.backup_replica.id
provider = aws.replica
target_bucket = "govuk-${var.govuk_environment}-aws-secondary-logging"
target_prefix = "s3/govuk-${var.govuk_environment}-database-backups-replica/"
}

resource "aws_s3_bucket_versioning" "backup_main" {
bucket = aws_s3_bucket.backup_main.id
versioning_configuration { status = "Enabled" }
}

role_name = "${local.db_backup_service_account_name}-${data.tfe_outputs.cluster_infrastructure.nonsensitive_values.cluster_id}"
role_description = "Role for database backup jobs. Corresponds to ${local.db_backup_service_account_name} k8s ServiceAccount."
max_session_duration = 28800
resource "aws_s3_bucket_versioning" "backup_replica" {
bucket = aws_s3_bucket.backup_replica.id
provider = aws.replica
versioning_configuration { status = "Enabled" }
}

role_policy_arns = { policy = aws_iam_policy.db_backup_s3.arn }
oidc_providers = {
main = {
provider_arn = data.tfe_outputs.cluster_infrastructure.nonsensitive_values.cluster_oidc_provider_arn
namespace_service_accounts = ["apps:${local.db_backup_service_account_name}"]
resource "aws_s3_bucket_lifecycle_configuration" "backup_main" {
bucket = aws_s3_bucket.backup_main.id
rule {
id = "production"
status = var.govuk_environment == "production" ? "Enabled" : "Disabled"
filter {}
transition {
days = 30
storage_class = "STANDARD_IA"
}
transition {
days = 60
storage_class = "GLACIER"
}
expiration { days = 120 }
noncurrent_version_expiration { noncurrent_days = 1 }
}
rule {
id = "non-production"
status = var.govuk_environment != "production" ? "Enabled" : "Disabled"
filter {}
expiration { days = 2 }
noncurrent_version_expiration { noncurrent_days = 1 }
}
}

resource "aws_s3_bucket_lifecycle_configuration" "backup_replica" {
bucket = aws_s3_bucket.backup_replica.id
provider = aws.replica
rule {
id = "production"
status = var.govuk_environment == "production" ? "Enabled" : "Disabled"
filter {}
transition {
days = 30
storage_class = "STANDARD_IA"
}
transition {
days = 60
storage_class = "GLACIER"
}
expiration { days = 120 }
noncurrent_version_expiration { noncurrent_days = 1 }
}
rule {
id = "non-production"
status = var.govuk_environment != "production" ? "Enabled" : "Disabled"
filter {}
expiration { days = 2 }
noncurrent_version_expiration { noncurrent_days = 1 }
}
}

data "aws_iam_policy_document" "db_backup_s3" {
resource "aws_s3_bucket_replication_configuration" "backup_main" {
depends_on = [aws_s3_bucket_versioning.backup_main] # TF doesn't infer this :(

bucket = aws_s3_bucket.backup_main.id
role = aws_iam_role.backup_replication.arn

rule {
id = "replicate-db-backups-out-of-region"
priority = 10
status = var.govuk_environment == "production" ? "Enabled" : "Disabled"
delete_marker_replication { status = "Disabled" }
destination {
bucket = aws_s3_bucket.backup_replica.arn
storage_class = "STANDARD_IA"
}
filter {}
}
}

data "aws_iam_policy_document" "backup_s3_can_assume_role" {
statement {
sid = "Read"
actions = ["sts:AssumeRole"]
principals {
type = "Service"
identifiers = ["s3.amazonaws.com"]
}
}
}

resource "aws_iam_role" "backup_replication" {
name = "database-backups-s3-replication"
assume_role_policy = data.aws_iam_policy_document.backup_s3_can_assume_role.json
}

data "aws_iam_policy_document" "backup_replication" {
statement {
sid = "ReplicateFromSourceBucket"
actions = [
"s3:GetBucketLocation",
"s3:ListBucket",
"s3:GetObject",
"s3:GetObject*Attributes",
]
resources = [
"arn:aws:s3:::govuk-${var.govuk_environment}-database-backups",
"arn:aws:s3:::govuk-${var.govuk_environment}-database-backups/*",
"arn:aws:s3:::${local.env_also_reads_from[var.govuk_environment]}",
"arn:aws:s3:::${local.env_also_reads_from[var.govuk_environment]}/*",
"s3:GetObject*",
"s3:GetReplicationConfiguration",
]
resources = [aws_s3_bucket.backup_main.arn, "${aws_s3_bucket.backup_main.arn}/*"]
}
statement {
sid = "Write"
actions = ["s3:*MultipartUpload*", "s3:PutObject"]
resources = ["arn:aws:s3:::govuk-${var.govuk_environment}-database-backups/*"]
sid = "ReplicateToDestinationBuckets"
actions = ["s3:ObjectOwnerOverrideToBucketOwner", "s3:Replicate*"]
resources = ["${aws_s3_bucket.backup_replica.arn}/*"]
}
}

resource "aws_iam_policy" "db_backup_s3" {
name = "db_backup_s3"
description = "Permissions over this environment's govuk-*-database-backups bucket."
policy = data.aws_iam_policy_document.db_backup_s3.json
resource "aws_iam_policy" "backup_replication" {
name = "db-backup-s3-replication"
policy = data.aws_iam_policy_document.backup_replication.json
description = "Allow S3 to replicate the database backup bucket out-of-region."
}

resource "aws_iam_role_policy_attachment" "backup_replication" {
role = aws_iam_role.backup_replication.name
policy_arn = aws_iam_policy.backup_replication.arn
}
74 changes: 74 additions & 0 deletions terraform/deployments/govuk-publishing-infrastructure/db_backup_s3_import.tf
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
import {
to = aws_s3_bucket.backup_main
id = "govuk-${var.govuk_environment}-database-backups"
}

import {
to = aws_s3_bucket.backup_replica
id = "govuk-${var.govuk_environment}-database-backups-replica"
}

/*import {
to = aws_s3_bucket_object_lock_configuration.backup_main
id = "govuk-${var.govuk_environment}-database-backups"
}
import {
to = aws_s3_bucket_object_lock_configuration.backup_replica
id = "govuk-${var.govuk_environment}-database-backups-replica"
}*/

import {
to = aws_s3_bucket_public_access_block.backup_main
id = "govuk-${var.govuk_environment}-database-backups"
}

import {
to = aws_s3_bucket_public_access_block.backup_replica
id = "govuk-${var.govuk_environment}-database-backups-replica"
}

import {
to = aws_s3_bucket_logging.backup_main
id = "govuk-${var.govuk_environment}-database-backups"
}

import {
to = aws_s3_bucket_logging.backup_replica
id = "govuk-${var.govuk_environment}-database-backups-replica"
}

import {
to = aws_s3_bucket_versioning.backup_main
id = "govuk-${var.govuk_environment}-database-backups"
}

import {
to = aws_s3_bucket_versioning.backup_replica
id = "govuk-${var.govuk_environment}-database-backups-replica"
}

import {
to = aws_s3_bucket_lifecycle_configuration.backup_main
id = "govuk-${var.govuk_environment}-database-backups"
}

import {
to = aws_s3_bucket_lifecycle_configuration.backup_replica
id = "govuk-${var.govuk_environment}-database-backups-replica"
}

import {
to = aws_s3_bucket_replication_configuration.backup_main
id = "govuk-${var.govuk_environment}-database-backups"
}

import {
to = aws_iam_role.backup_replication
id = "database-backups-s3-replication"
}

import {
to = aws_iam_policy.backup_replication
id = "arn:aws:iam::${data.aws_caller_identity.current.account_id}:policy/db-backup-s3-replication"
}
Loading

0 comments on commit c316686

Please sign in to comment.