Update consumer terraform

openclimatefix · Nov 9, 2023 · 53c4d60 · 53c4d60
1 parent 1e07c19
commit 53c4d60
Show file tree

Hide file tree

Showing 7 changed files with 313 additions and 258 deletions.
diff --git a/terraform/modules/services/nwp_consumer/README.md b/terraform/modules/services/nwp_consumer/README.md
@@ -1,8 +1,8 @@
-# Modules/Services/NWP
+# Modules/Services/nwp_consumer
 
 This module makes
-- AWS task definition
-- IAM role to setup application
-- IAM role for running task
-- get secrets for NWP API
-- temp: scheduled aws task, and iam roles
+- AWS ECS Task Definition, with:
+  - S3 and SecretsManager access
+  - AWS CloudWatch Logs group
+- IAM role to create the ECS Task
+- IAM role to run the ECS Task
diff --git a/terraform/modules/services/nwp_consumer/cloudwatch.tf b/terraform/modules/services/nwp_consumer/cloudwatch.tf
@@ -3,45 +3,47 @@
 # 2. IAM policy to allow read and write to cloudwatch logs
 
 locals {
-  log_group_name = "/aws/ecs/consumer/${var.app_name}/"
+  log_group_name = "/aws/ecs/${var.ecs-task_type}/${var.ecs-task_name}/"
 }
 
 # 1.
-resource "aws_cloudwatch_log_group" "nwp" {
+resource "aws_cloudwatch_log_group" "log_group" {
   name = local.log_group_name
 
   retention_in_days = 7
 
   tags = {
-    Environment = var.aws_config.environment
-    Application = "nowcasting"
+    Environment = var.aws-environment
+    Application = "ecs-${var.ecs-task_name}"
   }
 }
 
-# 2.
-resource "aws_iam_policy" "cloudwatch-nwp" {
-  name        = "${var.app_name}-cloudwatch-read-and-write"
-  path        = "/consumer/${var.app_name}/"
-  description = "Policy to allow read and write to cloudwatch logs"
+# Describe actions of IAM policy allowing cloudwatch read and write
+data "aws_iam_policy_document" "log_policy" {
+  statement {
+    effect = "Allow"
+
+    actions = [
+      "logs:PutLogEvents",
+      "logs:CreateLogStream",
+      "logs:CreateLogGroup",
+      "logs:DescribeLogStreams",
+      "logs:DescribeLogGroups",
+      "logs:DeleteLogGroup",
+      "logs:PutRetentionPolicy"
+    ]
 
-  # Terraform's "jsonencode" function converts a
-  # Terraform expression result to valid JSON syntax.
-  policy = jsonencode({
-    Version = "2012-10-17"
-    Statement = [
-      {
-        Action = [
-          "logs:PutLogEvents",
-          "logs:CreateLogStream",
-          "logs:CreateLogGroup",
-          "logs:DescribeLogStreams",
-          "logs:DescribeLogGroups",
-          "logs:DeleteLogGroup",
-          "logs:PutRetentionPolicy"
-        ]
-        Effect   = "Allow"
-        Resource = "arn:aws:logs:*:*:log-group:${local.log_group_name}*"
-      },
+    resources = [
+      "arn:aws:logs:*:*:log-group:${local.log_group_name}*",
     ]
-  })
+  }
+}
+
+# 2. Create IAM policy with above actions on created cloudwatch log group
+resource "aws_iam_policy" "log_policy" {
+  name        = "${var.ecs-task_name}-cloudwatch-read-and-write"
+  path        = "/${var.ecs-task_type}/${var.ecs-task_name}/"
+  description = "Policy to allow read and write to cloudwatch logs"
+
+  policy = data.aws_iam_policy_document.log_policy.json
 }
diff --git a/terraform/modules/services/nwp_consumer/ecs.tf b/terraform/modules/services/nwp_consumer/ecs.tf
@@ -1,39 +1,39 @@
 # Creates:
-# 1. ECS Task to run the Consumer
+# 1. ECS Task Definition
 
-# Create the ECS Task Definition
-resource "aws_ecs_task_definition" "nwp-task-definition" {
-  family                   = "${var.app_name}"
+# 1. Create the ECS Task Definition
+resource "aws_ecs_task_definition" "task_def" {
+  family                   = var.ecs-task_name
   requires_compatibilities = ["FARGATE"]
   network_mode             = "awsvpc"
 
   # specific values are needed -
   # https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task-cpu-memory-error.html
-  cpu    = 1024
-  memory = 5120
+  cpu    = var.ecs-task_cpu
+  memory = var.ecs-task_memory
 
   tags = {
-    name = "${var.app_name}-consumer"
+    name = "${var.ecs-task_name}-${var.ecs-task_type}"
     type = "ecs"
   }
 
   volume {
     name = "tmp"
   }
 
-  task_role_arn         = aws_iam_role.consumer-nwp-iam-role.arn
-  execution_role_arn    = aws_iam_role.ecs_task_execution_role.arn
+  task_role_arn         = aws_iam_role.run_task_role.arn
+  execution_role_arn    = aws_iam_role.create_task_role.arn
   container_definitions = jsonencode([
     {
-      name      = "${var.app_name}-consumer"
-      image     = "ghcr.io/openclimatefix/nwp-consumer:${var.docker_config.container_tag}"
+      name      = "${var.ecs-task_name}-${var.ecs-task_type}"
+      image     = "${var.container-registry}/${var.container-name}:${var.container-tag}"
       essential = true
 
-      environment : var.docker_config.environment_vars
-      command : var.docker_config.command
+      environment : var.container-env_vars
+      command : var.container-command
 
       secrets : [
-        for key in var.docker_config.secret_vars : {
+        for key in var.container-secret_vars : {
           name : key
           valueFrom : "${data.aws_secretsmanager_secret_version.current.arn}:${key}::"
         }
@@ -43,7 +43,7 @@ resource "aws_ecs_task_definition" "nwp-task-definition" {
         "logDriver" : "awslogs",
         "options" : {
           "awslogs-group" : local.log_group_name,
-          "awslogs-region" : var.aws_config.region,
+          "awslogs-region" : var.aws-region,
           "awslogs-stream-prefix" : "streaming"
         }
       }

diff --git a/terraform/modules/services/nwp_consumer/iam.tf b/terraform/modules/services/nwp_consumer/iam.tf
@@ -2,70 +2,63 @@
 # Execution role is used to deploy the task
 # Instance role is used to run the task
 
-resource "aws_iam_role" "ecs_task_execution_role" {
-  name = "${var.app_name}-execution-role"
-
-  assume_role_policy = <<EOF
-{
- "Version": "2012-10-17",
- "Statement": [
-   {
-     "Action": "sts:AssumeRole",
-     "Principal": {
-       "Service": "ecs-tasks.amazonaws.com"
-     },
-     "Effect": "Allow",
-     "Sid": ""
-   }
- ]
-}
-EOF
-}
-
-resource "aws_iam_role_policy_attachment" "ecs-task-execution-role-policy-attachment" {
-  role       = aws_iam_role.ecs_task_execution_role.name
-  policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy"
-}
-
-resource "aws_iam_role_policy_attachment" "attach-logs-execution" {
-  role       = aws_iam_role.ecs_task_execution_role.name
-  policy_arn = aws_iam_policy.cloudwatch-nwp.arn
-}
-
-data "aws_iam_policy_document" "ec2-instance-assume-role-policy" {
+data "aws_iam_policy_document" "ecs_assume_role_policy" {
+  version = "2012-10-17"
   statement {
-    actions = ["sts:AssumeRole"]
+    effect = "Allow"
 
     principals {
       type        = "Service"
       identifiers = ["ecs-tasks.amazonaws.com"]
     }
+
+    actions = ["sts:AssumeRole"]
   }
 }
 
-resource "aws_iam_role" "consumer-nwp-iam-role" {
-  name               = "consumer-${var.app_name}-iam-role"
-  path               = "/consumer/"
-  assume_role_policy = data.aws_iam_policy_document.ec2-instance-assume-role-policy.json
+// Create Task Role ------------------------------------------------------
+
+resource "aws_iam_role" "create_task_role" {
+  name = "${var.ecs-task_name}-execution-role"
+  assume_role_policy = data.aws_iam_policy_document.ecs_assume_role_policy.json
 }
 
-resource "aws_iam_role_policy_attachment" "attach-write-s3" {
-  role       = aws_iam_role.consumer-nwp-iam-role.name
-  policy_arn = var.s3_config.bucket_write_policy_arn
+resource "aws_iam_role_policy_attachment" "create_task_policy" {
+  role       = aws_iam_role.create_task_role.name
+  policy_arn = "arn:aws:iam::aws:policy/service-role/AmazonECSTaskExecutionRolePolicy"
 }
 
-resource "aws_iam_role_policy_attachment" "attach-logs" {
-  role       = aws_iam_role.consumer-nwp-iam-role.name
-  policy_arn = aws_iam_policy.cloudwatch-nwp.arn
+resource "aws_iam_role_policy_attachment" "create_logs_policy" {
+  role       = aws_iam_role.create_task_role.name
+  policy_arn = aws_iam_policy.log_policy.arn
 }
 
-resource "aws_iam_role_policy_attachment" "read-secret-execution" {
-  role       = aws_iam_role.ecs_task_execution_role.name
+resource "aws_iam_role_policy_attachment" "access_secret_policy" {
+  role       = aws_iam_role.create_task_role.name
   policy_arn = aws_iam_policy.secret_read_policy.arn
 }
 
-resource "aws_iam_role_policy_attachment" "read-secret" {
-  role       = aws_iam_role.consumer-nwp-iam-role.name
+// Run Task Role ---------------------------------------------------------
+
+resource "aws_iam_role" "run_task_role" {
+  name               = "${var.ecs-task_type}-${var.ecs-task_name}-iam-role"
+  path               = "/${var.ecs-task_type}/"
+  assume_role_policy = data.aws_iam_policy_document.ecs_assume_role_policy.json
+}
+
+resource "aws_iam_role_policy_attachment" "access_s3_policy" {
+  for_each   = var.s3-buckets
+  role       = aws_iam_role.run_task_role.name
+  policy_arn = each.value.access_policy_arn
+}
+
+resource "aws_iam_role_policy_attachment" "access_logs_policy" {
+  role       = aws_iam_role.run_task_role.name
+  policy_arn = aws_iam_policy.log_policy.arn
+}
+
+resource "aws_iam_role_policy_attachment" "access_secret_policy" {
+  role       = aws_iam_role.run_task_role.name
   policy_arn = aws_iam_policy.secret_read_policy.arn
 }
 
diff --git a/terraform/modules/services/nwp_consumer/secrets.tf b/terraform/modules/services/nwp_consumer/secrets.tf
@@ -1,32 +1,37 @@
-# Read in required secrets for consumer
-# Get the secret resource from AWS for each entry in the list
+# Access a secret in secrets manager
+# 1. Gets the id of current version of the secret
+# 2. Creates a policy to allow read access to the secret
+
+# Read in required secret for task
 data "aws_secretsmanager_secret" "secret" {
-  name = var.aws_config.secretsmanager_secret_name
+  name = var.aws-secretsmanager_secret_name
 }
 
-# Get the current secret value from AWS for the secret
+# 1. Get the current secret value from AWS for the secret
 data "aws_secretsmanager_secret_version" "current" {
   secret_id = data.aws_secretsmanager_secret.secret.id
 }
 
-# Get an IAM role to access the secret 
+# Create a policy enabling read access to secrets
+data "aws_iam_policy_document" "secret_read_policy" {
+  statement {
+    version = "2012-10-17"
+    actions = [
+      "secretsmanager:ListSecretVersionIds",
+      "secretsmanager:GetSecretValue",
+    ]
+    effect = "Allow"
+    resources = [
+      data.aws_secretsmanager_secret_version.current.arn,
+    ]
+  }
+}
+
+# 2. Create an IAM policy to access the current version of the secret
 resource "aws_iam_policy" "secret_read_policy" {
-  name        = "${var.app_name}-secret-read-policy"
-  path        = "/consumer/nwp/"
+  name        = "${var.ecs-task_name}-secret-read-policy"
+  path        = "/${var.ecs-task_type}/${var.ecs-task_name}/"
   description = "Policy to allow read access to secret."
 
-  policy = jsonencode({
-    Version = "2012-10-17"
-    Statement = [
-      {
-        Action = [
-          "secretsmanager:ListSecretVersionIds",
-          "secretsmanager:GetSecretValue",
-        ]
-        Effect   = "Allow"
-        Resource = data.aws_secretsmanager_secret_version.current.arn
-      },
-    ]
-  })
+  policy = data.aws_iam_policy_document.secret_read_policy.json
 }
-