SP-2061 - Update terraform pipeline to plan on PR changes and include…

… a permanent development environment #minor (#262)

* SP-2061 - Update terraform pipeline to plan on PR changes and include a permanent development environment #minor
* SP-2061 - Update AWS, fix deprecations, add tflint #minor
* SP-2061 - Remove db_subnet_group as actually managed in opg-shared-infrastructure #minor
* SP-2061 - Fix ALB Bucket S3 Permissions, wait for ECS Service to be stable #minor

.github/workflows/terraform.yml

@@ -0,0 +1,132 @@
+name: Terraform Lint, Plan, Apply
+
+on:  
+  pull_request:
+    branches:
+      - main
+    paths:
+      - 'terraform/*'
+  push:
+    branches:
+      - main
+  workflow_dispatch:
+
+defaults:
+  run:
+    shell: bash
+    working-directory: terraform
+
+jobs:
+  pull-tag:
+    name: Pull latest tag from parameter store.
+    runs-on: ubuntu-latest
+    outputs:
+      latest-tag: ${{ steps.output_tag.outputs.tag }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: eu-west-1
+          role-to-assume: arn:aws:iam::997462338508:role/incident-response-ci
+          role-duration-seconds: 3600
+          role-session-name: GitHubActions
+      - name: Install AWS CLI
+        id: install-aws-cli
+        uses: unfor19/install-aws-cli-action@v1
+      - name: Pull Tag from Parameter Store
+        run: |
+          echo  'TAG_NAME='$(aws ssm get-parameter --region "eu-west-1" --name "incident-response-production-tag" --query Parameter.Value) >> $GITHUB_ENV
+      - name: Output Tag
+        id: output_tag
+        run: echo "::set-output name=tag::${{ env.TAG_NAME }}"
+
+  lint-and-validate:
+    name: Terraform Lint & Validate
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: hashicorp/setup-terraform@v1
+        with:
+          terraform_version: 1.8.1
+          terraform_wrapper: false
+      - uses: terraform-linters/setup-tflint@v4
+        name: Setup TFLint
+        with:
+          tflint_version: v0.50.1
+
+      - name: Configure AWS Credentials For Terraform
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: eu-west-1
+          role-session-name: GitHubActionsTerraform
+
+      - name: Terraform Format
+        run: terraform fmt --check --recursive 
+
+      - name: TF Lint
+        run: tflint --recursive
+
+      - name: Terraform Init
+        run: terraform init
+
+      - name: Terraform Validate
+        run: terraform validate
+
+
+  plan-and-apply:
+    name: Plan ${{ matrix.environment }}
+    runs-on: ubuntu-latest
+    needs: 
+      - lint-and-validate
+      - pull-tag
+    env:
+      TF_VAR_response_tag: ${{ needs.pull-tag.outputs.latest-tag }}
+      TF_VAR_nginx_tag: ${{ needs.pull-tag.outputs.latest-tag }}
+    strategy:
+      max-parallel: 1
+      matrix:
+        include:
+          - environment: "Development"
+            workspace_environment: "development"
+
+          - environment: "Production"
+            workspace_environment: "production"
+
+    steps:
+      - uses: actions/checkout@v4
+      - uses: hashicorp/setup-terraform@v1
+        with:
+          terraform_version: 1.8.1
+          terraform_wrapper: false
+
+      - name: Configure AWS Credentials For Terraform
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          aws-region: eu-west-1
+          role-session-name: GitHubActionsTerraform
+
+      - name: Setup
+        run: echo TF_WORKSPACE=${{ matrix.workspace_environment }} >> $GITHUB_ENV
+
+      - name: Init
+        run: terraform init
+
+      - name: Plan
+        run: terraform plan --lock-timeout=300s --parallelism=200 --out=${{ env.TF_WORKSPACE }}.plan > ${{ env.TF_WORKSPACE }}.log
+
+      - name: Output Plan
+        run: cat ${{ env.TF_WORKSPACE }}.log
+
+      - name: Output ConcisePlan
+        run: cat ${{ env.TF_WORKSPACE }}.log | grep '\.' | grep '#' || true
+
+      - name: Apply ${{ matrix.environment }}
+        if: github.ref == 'refs/heads/main'
+        run:  terraform apply -parallelism=200 -lock-timeout=300s ${{ env.TF_WORKSPACE }}.plan

terraform/access_logs.tf

@@ -27,9 +27,24 @@ resource "aws_s3_bucket" "access_log" {
   force_destroy = true
 }
 
-resource "aws_s3_bucket_acl" "access_log" {
+resource "aws_s3_bucket_ownership_controls" "bucket_object_ownership" {
   bucket = aws_s3_bucket.access_log.id
-  acl    = "private"
+  rule {
+    object_ownership = "BucketOwnerEnforced"
+  }
+}
+
+resource "aws_s3_bucket_lifecycle_configuration" "bucket" {
+  bucket = aws_s3_bucket.access_log.id
+
+  rule {
+    id     = "ExpireObjectsAfter13Months"
+    status = "Enabled"
+
+    expiration {
+      days = 400
+    }
+  }
 }
 
 resource "aws_s3_bucket_server_side_encryption_configuration" "access_log" {
@@ -42,6 +57,15 @@ resource "aws_s3_bucket_server_side_encryption_configuration" "access_log" {
   }
 }
 
+resource "aws_s3_bucket_public_access_block" "public_access_policy" {
+  bucket = aws_s3_bucket.access_log.id
+
+  block_public_acls       = true
+  block_public_policy     = true
+  ignore_public_acls      = true
+  restrict_public_buckets = true
+}
+
 resource "aws_s3_bucket_policy" "access_log" {
   bucket = aws_s3_bucket.access_log.id
   policy = data.aws_iam_policy_document.loadbalancer.json

terraform/aurora.tf

@@ -42,8 +42,3 @@ resource "aws_security_group_rule" "response_rds_ecs_task" {
   source_security_group_id = aws_security_group.ecs_service.id
   description              = "Response RDS inbound from Response ECS tasks"
 }
-
-resource "aws_db_subnet_group" "data_persitance_subnet_group" {
-  name       = "data-persitance-subnet-${terraform.workspace}"
-  subnet_ids = data.aws_subnet_ids.data_persitance.ids
-}

terraform/data_sources.tf

@@ -2,38 +2,27 @@ data "aws_vpc" "default" {
   default = true
 }
 
-data "aws_availability_zones" "available" {
-  state = "available"
-}
-
-data "aws_subnet_ids" "public" {
-  vpc_id = data.aws_vpc.default.id
-  tags   = { Name = "*public*" }
-}
-
-data "aws_subnet" "public" {
-  count             = length(tolist(data.aws_subnet_ids.public.ids))
-  availability_zone = data.aws_availability_zones.available.names[count.index]
-  tags              = { Name = "*public*" }
-}
-
-data "aws_subnet_ids" "private" {
-  vpc_id = data.aws_vpc.default.id
-  tags   = { Name = "private" }
-}
+data "aws_subnets" "public" {
+  filter {
+    name   = "vpc-id"
+    values = [data.aws_vpc.default.id]
+  }
 
-data "aws_subnet" "private" {
-  count             = length(tolist(data.aws_subnet_ids.private.ids))
-  availability_zone = data.aws_availability_zones.available.names[count.index]
-  tags              = { Name = "private" }
+  filter {
+    name   = "tag:Name"
+    values = ["public"]
+  }
 }
 
-data "aws_subnet_ids" "data_persitance" {
-  vpc_id = data.aws_vpc.default.id
+data "aws_subnets" "private" {
+  filter {
+    name   = "vpc-id"
+    values = [data.aws_vpc.default.id]
+  }
 
   filter {
     name   = "tag:Name"
-    values = ["persistence"]
+    values = ["private"]
   }
 }