VEGA-2289 - S3 Storage & Replication #minor (#103)

* VEGA-2289 - Add local localstack config and local lpa-store-static bucket #minor
* VEGA-2289 - Add initial cross region S3 setup #minor
* VEGA-2289 - Remove redundant keys and sort permissions for cross region replication #minor
* VEGA-2289 - Add Accces Logging to ccount Buckets #minor
* VEGA-2289 - Add Cross Account S3 Backup #minor
* VEGA-2289 - Grant Lambda Roles S3 & KMS Permissions for Object Put #minor
* VEGA-2289 - Move lambda policy rather than destroy/create #patch
* VEGA-2289 - Remove Life Cycle configuration from S3 Modules as we want to keep everything #minor

docker-compose.yml

@@ -8,7 +8,7 @@ services:
       - "8000:8000"
 
   lambda-create:
-    depends_on: [ddb]
+    depends_on: [ ddb ]
     build:
       context: .
       dockerfile: ./lambda/Dockerfile
@@ -27,7 +27,7 @@ services:
     entrypoint: /aws-lambda/aws-lambda-rie /var/task/main
 
   lambda-update:
-    depends_on: [ddb]
+    depends_on: [ ddb ]
     build:
       context: .
       dockerfile: ./lambda/Dockerfile
@@ -46,7 +46,7 @@ services:
     entrypoint: /aws-lambda/aws-lambda-rie /var/task/main
 
   lambda-get:
-    depends_on: [ddb]
+    depends_on: [ ddb ]
     build:
       context: .
       dockerfile: ./lambda/Dockerfile
@@ -65,15 +65,15 @@ services:
     entrypoint: /aws-lambda/aws-lambda-rie /var/task/main
 
   apigw:
-    depends_on: [lambda-create, lambda-update, lambda-get]
+    depends_on: [ lambda-create, lambda-update, lambda-get ]
     build:
       context: .
       dockerfile: ./mock-apigw/Dockerfile
     ports:
       - 9000:8080
 
   aws:
-    depends_on: [ddb]
+    depends_on: [ ddb ]
     image: amazon/aws-cli:latest
     environment:
       AWS_ENDPOINT_URL: http://ddb:8000/
@@ -82,6 +82,19 @@ services:
       AWS_SECRET_ACCESS_KEY: X
       AWS_PAGER: ""
 
+  localstack:
+    image: localstack/localstack:3.0
+    volumes:
+      - "./localstack/init:/etc/localstack/init/ready.d"
+      - "./localstack/wait:/scripts/wait"
+    environment:
+      AWS_DEFAULT_REGION: eu-west-1
+    healthcheck:
+      test: bash /scripts/wait/healthcheck.sh
+      interval: 10s
+      timeout: 10s
+      retries: 50
+
   go-lint:
     image: golangci/golangci-lint:v1.55.2
     working_dir: /go/src/app

localstack/init/localstack_init.sh

@@ -0,0 +1,33 @@
+#! /usr/bin/env bash
+create_bucket() {
+    BUCKET=$1
+    # Create Private Bucket
+    awslocal s3api create-bucket \
+        --acl private \
+        --region eu-west-1 \
+        --create-bucket-configuration LocationConstraint=eu-west-1 \
+        --bucket "$BUCKET"
+
+    # Add Public Access Block
+    awslocal s3api put-public-access-block \
+        --public-access-block-configuration "BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true" \
+        --bucket "$BUCKET"
+
+    # Add Default Encryption
+    awslocal s3api put-bucket-encryption \
+        --bucket "$BUCKET" \
+        --server-side-encryption-configuration '{ "Rules": [ { "ApplyServerSideEncryptionByDefault": { "SSEAlgorithm": "AES256" } } ] }'
+
+    # Add Encryption Policy
+    awslocal s3api put-bucket-policy \
+        --policy '{ "Statement": [ { "Sid": "DenyUnEncryptedObjectUploads", "Effect": "Deny", "Principal": { "AWS": "*" }, "Action": "s3:PutObject", "Resource": "arn:aws:s3:::'${BUCKET}'/*", "Condition":  { "StringNotEquals": { "s3:x-amz-server-side-encryption": "AES256" } } }, { "Sid": "DenyUnEncryptedObjectUploads", "Effect": "Deny", "Principal": { "AWS": "*" }, "Action": "s3:PutObject", "Resource": "arn:aws:s3:::'${BUCKET}'/*", "Condition":  { "Bool": { "aws:SecureTransport": false } } } ] }' \
+        --bucket "$BUCKET"
+
+    # Add Bucket Versioning
+    awslocal s3api put-bucket-versioning \
+        --versioning-configuration '{ "MFADelete": "Disabled", "Status": "Enabled" }' \
+        --bucket "$BUCKET"
+}
+
+# S3
+create_bucket "opg-lpa-store-static-eu-west-1"

localstack/wait/healthcheck.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+# S3
+buckets=$(awslocal s3 ls)
+
+echo $buckets | grep "opg-lpa-store-static-eu-west-1" || exit 1

terraform/environment/.envrc

@@ -1,5 +1,6 @@
 # Terraform
 export TF_WORKSPACE=development
+export TF_VAR_app_version=latest
 export TF_VAR_default_role=operator
 export TF_VAR_management_role=operator
 

terraform/environment/data_sources.tf

@@ -0,0 +1,3 @@
+data "aws_caller_identity" "current" {
+  provider = aws.eu_west_1
+}

terraform/environment/region/iam.tf

@@ -0,0 +1,70 @@
+moved {
+  from = aws_iam_role_policy.lambda
+  to   = aws_iam_role_policy.lambda_dynamodb
+}
+
+resource "aws_iam_role_policy" "lambda_dynamodb" {
+  for_each = local.functions
+  name     = "LambdaAllowDynamoDB"
+  role     = module.lambda[each.key].iam_role.id
+  policy   = data.aws_iam_policy_document.lambda_dynamodb_policy.json
+  provider = aws.region
+}
+
+data "aws_iam_policy_document" "lambda_dynamodb_policy" {
+  statement {
+    sid       = "allowDynamoDB"
+    effect    = "Allow"
+    resources = [var.dynamodb_arn]
+    actions = [
+      "dynamodb:PutItem",
+      "dynamodb:GetItem",
+    ]
+  }
+}
+
+resource "aws_iam_role_policy" "lambda_s3" {
+  for_each = local.functions
+  name     = "LambdaAllowS3"
+  role     = module.lambda[each.key].iam_role.id
+  policy   = data.aws_iam_policy_document.lambda_s3_policy.json
+  provider = aws.region
+}
+
+data "aws_iam_policy_document" "lambda_s3_policy" {
+  statement {
+    sid    = "allowS3Access"
+    effect = "Allow"
+    resources = [
+      var.lpa_store_static_bucket.arn,
+      "${var.lpa_store_static_bucket.arn}/*",
+    ]
+    actions = [
+      "s3:PutObject",
+    ]
+  }
+  statement {
+    sid       = "allowS3KMS"
+    effect    = "Allow"
+    resources = [var.lpa_store_static_bucket_kms_key.arn]
+    actions = [
+      "kms:GenerateDataKey",
+      "kms:Encrypt"
+    ]
+
+    condition {
+      test     = "StringLike"
+      variable = "kms:ViaService"
+      values   = ["s3.${data.aws_region.current.name}.amazonaws.com"]
+    }
+
+    condition {
+      test     = "StringLike"
+      variable = "kms:EncryptionContext:aws:s3:arn"
+      values = [
+        "${var.lpa_store_static_bucket.arn}/*",
+      ]
+    }
+  }
+}
+

terraform/environment/region/main.tf → terraform/environment/region/lambda.tf

@@ -16,9 +16,9 @@ module "lambda" {
   cloudwatch_kms_key_id = aws_kms_key.cloudwatch.arn
 
   environment_variables = {
-    DDB_TABLE_NAME_DEEDS = var.dynamodb_name
+    DDB_TABLE_NAME_DEEDS   = var.dynamodb_name
     DDB_TABLE_NAME_CHANGES = var.dynamodb_name_changes
-    JWT_SECRET_KEY = "secret"
+    JWT_SECRET_KEY         = "secret"
   }
 
   providers = {
@@ -31,23 +31,3 @@ data "aws_ecr_repository" "lambda" {
   name     = "lpa-store/lambda/api-${each.key}"
   provider = aws.management
 }
-
-resource "aws_iam_role_policy" "lambda" {
-  for_each = local.functions
-  name     = "LambdaAllowDynamoDB"
-  role     = module.lambda[each.key].iam_role_id
-  policy   = data.aws_iam_policy_document.lambda_access_ddb.json
-  provider = aws.region
-}
-
-data "aws_iam_policy_document" "lambda_access_ddb" {
-  statement {
-    sid       = "allowDynamoDB"
-    effect    = "Allow"
-    resources = [var.dynamodb_arn]
-    actions = [
-      "dynamodb:PutItem",
-      "dynamodb:GetItem",
-    ]
-  }
-}

terraform/environment/region/outputs.tf

@@ -1,3 +1,9 @@
 output "base_url" {
   value = "https://${local.domain_name}"
 }
+
+output "lambda_iam_roles" {
+  value = [
+    for lambda in module.lambda : lambda.iam_role
+  ]
+}

terraform/environment/region/variables.tf

@@ -1,13 +1,19 @@
-variable "environment_name" {
-  description = "The name of the environment the region is deployed to"
-  type        = string
+variable "allowed_arns" {
+  description = "List of external ARNs allowed to access the API Gateway"
+  type        = list(string)
 }
 
 variable "app_version" {
   description = "Version of application to deploy"
   type        = string
 }
 
+variable "dns_weighting" {
+  description = "What percentage of DNS traffic to send to this region"
+  type        = number
+  default     = 50
+}
+
 variable "dynamodb_arn" {
   description = "ARN of DynamoDB table"
   type        = string
@@ -28,13 +34,15 @@ variable "dynamodb_name_changes" {
   type        = string
 }
 
-variable "allowed_arns" {
-  description = "List of external ARNs allowed to access the API Gateway"
-  type        = list(string)
+variable "environment_name" {
+  description = "The name of the environment the region is deployed to"
+  type        = string
 }
 
-variable "dns_weighting" {
-  description = "What percentage of DNS traffic to send to this region"
-  type        = number
-  default     = 50
+variable "lpa_store_static_bucket" {
+  description = "LPA Store Static bucket object for the region"
+}
+
+variable "lpa_store_static_bucket_kms_key" {
+  description = "LPA Store Static bucket KMS Key object for the region"
 }

terraform/environment/regions.tf

@@ -1,14 +1,16 @@
 module "eu_west_1" {
   source = "./region"
 
-  app_version           = var.app_version
-  dynamodb_arn          = aws_dynamodb_table.deeds_table.arn
-  dynamodb_name         = aws_dynamodb_table.deeds_table.name
-  dynamodb_arn_changes  = aws_dynamodb_table.changes_table.arn
-  dynamodb_name_changes = aws_dynamodb_table.changes_table.name
-  environment_name      = local.environment_name
-  allowed_arns          = local.environment.allowed_arns
-  dns_weighting         = 100
+  allowed_arns                    = local.environment.allowed_arns
+  app_version                     = var.app_version
+  dns_weighting                   = 100
+  dynamodb_arn                    = aws_dynamodb_table.deeds_table.arn
+  dynamodb_arn_changes            = aws_dynamodb_table.changes_table.arn
+  dynamodb_name                   = aws_dynamodb_table.deeds_table.name
+  dynamodb_name_changes           = aws_dynamodb_table.changes_table.name
+  environment_name                = local.environment_name
+  lpa_store_static_bucket         = module.s3_lpa_store_static_eu_west_1.bucket
+  lpa_store_static_bucket_kms_key = module.s3_lpa_store_static_eu_west_1.encryption_kms_key
 
   providers = {
     aws.region     = aws.eu_west_1
@@ -19,14 +21,16 @@ module "eu_west_1" {
 module "eu_west_2" {
   source = "./region"
 
-  app_version           = var.app_version
-  dynamodb_arn          = aws_dynamodb_table_replica.deeds_table.arn
-  dynamodb_name         = aws_dynamodb_table.deeds_table.name
-  dynamodb_arn_changes  = aws_dynamodb_table_replica.changes_table.arn
-  dynamodb_name_changes = aws_dynamodb_table.changes_table.name
-  environment_name      = local.environment_name
-  allowed_arns          = local.environment.allowed_arns
-  dns_weighting         = 0
+  allowed_arns                    = local.environment.allowed_arns
+  app_version                     = var.app_version
+  dns_weighting                   = 0
+  dynamodb_arn                    = aws_dynamodb_table_replica.deeds_table.arn
+  dynamodb_arn_changes            = aws_dynamodb_table_replica.changes_table.arn
+  dynamodb_name                   = aws_dynamodb_table.deeds_table.name
+  dynamodb_name_changes           = aws_dynamodb_table.changes_table.name
+  environment_name                = local.environment_name
+  lpa_store_static_bucket         = module.s3_lpa_store_static_eu_west_2.bucket
+  lpa_store_static_bucket_kms_key = module.s3_lpa_store_static_eu_west_2.encryption_kms_key
 
   providers = {
     aws.region     = aws.eu_west_2