From b5de11c84d2f3d0b3ea3f3369e5b500f2c81ab46 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Thu, 5 Dec 2024 17:56:39 +0100 Subject: [PATCH 1/5] feat: add ecs variables for active campaign --- apps/infrastructure/src/modules/cms/ecs.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/infrastructure/src/modules/cms/ecs.tf b/apps/infrastructure/src/modules/cms/ecs.tf index dd32b3f48..74e104bae 100644 --- a/apps/infrastructure/src/modules/cms/ecs.tf +++ b/apps/infrastructure/src/modules/cms/ecs.tf @@ -15,7 +15,7 @@ resource "aws_ecs_task_definition" "cms_task_def" { memory = var.cms_app_memory container_definitions = templatefile( "${path.module}/task-definitions/cms_app.json.tpl", - { + merge({ image = module.ecr.repository_url fargate_cpu = var.cms_app_cpu fargate_memory = var.cms_app_memory From 4a9da282dbd8e5469e3848d76ba8a5e7e01209c4 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Thu, 9 Jan 2025 11:18:12 +0100 Subject: [PATCH 2/5] feat: active campaign resync lambda implemented --- .changeset/lemon-dancers-teach.md | 5 +++ .../cloudwatch_metrics_alarms.tf | 6 +-- .../src/modules/active_campaign/iam.tf | 12 ++++- .../modules/active_campaign/lambda_resync.tf | 44 +++++++++++++++++++ .../{lambda.tf => lambda_sync.tf} | 2 +- .../src/modules/active_campaign/sqs.tf | 13 ++++++ apps/infrastructure/src/modules/cms/ecs.tf | 2 +- 7 files changed, 78 insertions(+), 6 deletions(-) create mode 100644 .changeset/lemon-dancers-teach.md create mode 100644 apps/infrastructure/src/modules/active_campaign/lambda_resync.tf rename apps/infrastructure/src/modules/active_campaign/{lambda.tf => lambda_sync.tf} (98%) diff --git a/.changeset/lemon-dancers-teach.md b/.changeset/lemon-dancers-teach.md new file mode 100644 index 000000000..959932827 --- /dev/null +++ b/.changeset/lemon-dancers-teach.md @@ -0,0 +1,5 @@ +--- +"infrastructure": minor +--- + +Active campaign resync lambda implemented diff --git a/apps/infrastructure/src/modules/active_campaign/cloudwatch_metrics_alarms.tf b/apps/infrastructure/src/modules/active_campaign/cloudwatch_metrics_alarms.tf index d0452a3b9..d2c08b5e4 100644 --- a/apps/infrastructure/src/modules/active_campaign/cloudwatch_metrics_alarms.tf +++ b/apps/infrastructure/src/modules/active_campaign/cloudwatch_metrics_alarms.tf @@ -17,8 +17,8 @@ resource "aws_cloudwatch_metric_alarm" "pipe_failed" { alarm_actions = [aws_sns_topic.alerts.arn] } -resource "aws_cloudwatch_metric_alarm" "dlq" { - alarm_name = "${local.prefix}-sqs-messages-in-dlq" +resource "aws_cloudwatch_metric_alarm" "resync_dlq" { + alarm_name = "${local.prefix}-sqs-messages-in-resync-dlq" comparison_operator = "GreaterThanOrEqualToThreshold" evaluation_periods = 1 period = 60 @@ -27,7 +27,7 @@ resource "aws_cloudwatch_metric_alarm" "dlq" { metric_name = "ApproximateNumberOfMessagesVisible" namespace = "AWS/SQS" dimensions = { - QueueName = aws_sqs_queue.fifo_dlq_queue.name + QueueName = aws_sqs_queue.fifo_resync_dlq_queue.name } alarm_description = "This metric monitors messages put in the dead letter queue" insufficient_data_actions = [] diff --git a/apps/infrastructure/src/modules/active_campaign/iam.tf b/apps/infrastructure/src/modules/active_campaign/iam.tf index 6dc67e885..b2a043599 100644 --- a/apps/infrastructure/src/modules/active_campaign/iam.tf +++ b/apps/infrastructure/src/modules/active_campaign/iam.tf @@ -68,7 +68,7 @@ resource "aws_iam_policy" "lambda_policy" { { Action = ["sqs:ReceiveMessage", "sqs:DeleteMessage", "sqs:GetQueueAttributes"], Effect = "Allow", - Resource = [aws_sqs_queue.fifo_queue.arn, aws_sqs_queue.fifo_dlq_queue.arn] + Resource = [aws_sqs_queue.fifo_queue.arn, aws_sqs_queue.fifo_dlq_queue.arn, aws_sqs_queue.fifo_resync_dlq_queue.arn] }, { Action = ["logs:CreateLogGroup", "logs:CreateLogStream", "logs:PutLogEvents"], @@ -92,4 +92,14 @@ resource "aws_iam_role_policy_attachment" "lambda_policy_attach" { resource "aws_iam_role_policy_attachment" "lambda_cognito_policy_attach" { role = module.lambda_sync.lambda_role_name policy_arn = "arn:aws:iam::aws:policy/AmazonCognitoReadOnly" +} + +resource "aws_iam_role_policy_attachment" "lambda_resync_policy_attach" { + role = module.lambda_resync.lambda_role_name + policy_arn = aws_iam_policy.lambda_policy.arn +} + +resource "aws_iam_role_policy_attachment" "lambda_resync_cognito_policy_attach" { + role = module.lambda_resync.lambda_role_name + policy_arn = "arn:aws:iam::aws:policy/AmazonCognitoReadOnly" } \ No newline at end of file diff --git a/apps/infrastructure/src/modules/active_campaign/lambda_resync.tf b/apps/infrastructure/src/modules/active_campaign/lambda_resync.tf new file mode 100644 index 000000000..fe646d013 --- /dev/null +++ b/apps/infrastructure/src/modules/active_campaign/lambda_resync.tf @@ -0,0 +1,44 @@ +# Lambda Function for SQS FIFO +module "lambda_resync" { + source = "git::github.com/terraform-aws-modules/terraform-aws-lambda.git?ref=9633abb6b6d275d3a28604dbfa755098470420d4" # v6.5.0 + + function_name = "${local.prefix}-resync-lambda" + description = "Lambda function that resyncs Active Campaign failed events" + + environment_variables = { + AC_API_KEY_PARAM = module.active_campaign_api_key.ssm_parameter_name + AC_BASE_URL_PARAM = module.active_campaign_base_url.ssm_parameter_name + COGNITO_USER_POOL_ID = var.cognito_user_pool.id + } + + runtime = "nodejs20.x" + architectures = ["x86_64"] + + handler = "index.resyncQueue" + source_path = "${path.module}/functions" + ignore_source_code_hash = true + create_current_version_allowed_triggers = false + + timeout = 120 + memory_size = 256 + maximum_retry_attempts = 3 + + event_source_mapping = { + sqs = { + event_source_arn = aws_sqs_queue.fifo_dlq_queue.arn + batch_size = 1 + scaling_config = { + maximum_concurrency = 2 + } + } + } + + allowed_triggers = { + sqs = { + principal = "sqs.amazonaws.com" + source_arn = aws_sqs_queue.fifo_dlq_queue.arn + } + } + + tags = var.tags +} diff --git a/apps/infrastructure/src/modules/active_campaign/lambda.tf b/apps/infrastructure/src/modules/active_campaign/lambda_sync.tf similarity index 98% rename from apps/infrastructure/src/modules/active_campaign/lambda.tf rename to apps/infrastructure/src/modules/active_campaign/lambda_sync.tf index 3834d82e7..9c2619b63 100644 --- a/apps/infrastructure/src/modules/active_campaign/lambda.tf +++ b/apps/infrastructure/src/modules/active_campaign/lambda_sync.tf @@ -19,7 +19,7 @@ module "lambda_sync" { ignore_source_code_hash = true create_current_version_allowed_triggers = false - timeout = 15 + timeout = 30 memory_size = 256 maximum_retry_attempts = 0 diff --git a/apps/infrastructure/src/modules/active_campaign/sqs.tf b/apps/infrastructure/src/modules/active_campaign/sqs.tf index b5746ce95..91c90dd26 100644 --- a/apps/infrastructure/src/modules/active_campaign/sqs.tf +++ b/apps/infrastructure/src/modules/active_campaign/sqs.tf @@ -5,6 +5,7 @@ resource "aws_sqs_queue" "fifo_queue" { content_based_deduplication = true deduplication_scope = "queue" fifo_throughput_limit = "perQueue" + visibility_timeout_seconds = 40 redrive_policy = jsonencode({ deadLetterTargetArn = aws_sqs_queue.fifo_dlq_queue.arn @@ -60,4 +61,16 @@ resource "aws_sqs_queue" "fifo_queue" { resource "aws_sqs_queue" "fifo_dlq_queue" { name = "${local.prefix}-events-dlq.fifo" fifo_queue = true + visibility_timeout_seconds = 360 + + redrive_policy = jsonencode({ + deadLetterTargetArn = aws_sqs_queue.fifo_resync_dlq_queue.arn + maxReceiveCount = 1 + }) +} + +# Dead Letter Queue (DLQ) +resource "aws_sqs_queue" "fifo_resync_dlq_queue" { + name = "${local.prefix}-resync-events-dlq.fifo" + fifo_queue = true } diff --git a/apps/infrastructure/src/modules/cms/ecs.tf b/apps/infrastructure/src/modules/cms/ecs.tf index 74e104bae..dd32b3f48 100644 --- a/apps/infrastructure/src/modules/cms/ecs.tf +++ b/apps/infrastructure/src/modules/cms/ecs.tf @@ -15,7 +15,7 @@ resource "aws_ecs_task_definition" "cms_task_def" { memory = var.cms_app_memory container_definitions = templatefile( "${path.module}/task-definitions/cms_app.json.tpl", - merge({ + { image = module.ecr.repository_url fargate_cpu = var.cms_app_cpu fargate_memory = var.cms_app_memory From 36a6d36ba53834d047ff527b7c8043ab4eff6ddd Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Thu, 9 Jan 2025 11:19:26 +0100 Subject: [PATCH 3/5] feat: github actions now also deploy ac resync lambda --- .../{deploy_ac_sync_lambda.yaml => deploy_ac_lambdas.yaml} | 6 ++++++ 1 file changed, 6 insertions(+) rename .github/workflows/{deploy_ac_sync_lambda.yaml => deploy_ac_lambdas.yaml} (86%) diff --git a/.github/workflows/deploy_ac_sync_lambda.yaml b/.github/workflows/deploy_ac_lambdas.yaml similarity index 86% rename from .github/workflows/deploy_ac_sync_lambda.yaml rename to .github/workflows/deploy_ac_lambdas.yaml index ca70c278b..fda46bf21 100644 --- a/.github/workflows/deploy_ac_sync_lambda.yaml +++ b/.github/workflows/deploy_ac_lambdas.yaml @@ -75,4 +75,10 @@ jobs: run: | aws lambda update-function-code \ --function-name ac-${{ github.event.inputs.environment || 'dev' }}-sync-lambda \ + --zip-file fileb://packages/active-campaign-client/target/function.zip + + - name: Deploy Lambda function (${{ github.event.inputs.environment || 'dev' }}) + run: | + aws lambda update-function-code \ + --function-name ac-${{ github.event.inputs.environment || 'dev' }}-resync-lambda \ --zip-file fileb://packages/active-campaign-client/target/function.zip \ No newline at end of file From a4dc87fc0c8d92c423491f553c34d7acb7a00f5a Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Thu, 9 Jan 2025 11:20:39 +0100 Subject: [PATCH 4/5] fix: revert github action name --- .../{deploy_ac_lambdas.yaml => deploy_ac_sync_lambda.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{deploy_ac_lambdas.yaml => deploy_ac_sync_lambda.yaml} (100%) diff --git a/.github/workflows/deploy_ac_lambdas.yaml b/.github/workflows/deploy_ac_sync_lambda.yaml similarity index 100% rename from .github/workflows/deploy_ac_lambdas.yaml rename to .github/workflows/deploy_ac_sync_lambda.yaml From 57be9145df2c808978dd47f8f0ba2e2f5ab4d058 Mon Sep 17 00:00:00 2001 From: christian-calabrese Date: Thu, 9 Jan 2025 11:23:44 +0100 Subject: [PATCH 5/5] chore: ran pre-commit --- apps/infrastructure/src/modules/active_campaign/sqs.tf | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/apps/infrastructure/src/modules/active_campaign/sqs.tf b/apps/infrastructure/src/modules/active_campaign/sqs.tf index 91c90dd26..00e9e0401 100644 --- a/apps/infrastructure/src/modules/active_campaign/sqs.tf +++ b/apps/infrastructure/src/modules/active_campaign/sqs.tf @@ -5,7 +5,7 @@ resource "aws_sqs_queue" "fifo_queue" { content_based_deduplication = true deduplication_scope = "queue" fifo_throughput_limit = "perQueue" - visibility_timeout_seconds = 40 + visibility_timeout_seconds = 40 redrive_policy = jsonencode({ deadLetterTargetArn = aws_sqs_queue.fifo_dlq_queue.arn @@ -59,10 +59,10 @@ resource "aws_sqs_queue" "fifo_queue" { # Dead Letter Queue (DLQ) resource "aws_sqs_queue" "fifo_dlq_queue" { - name = "${local.prefix}-events-dlq.fifo" - fifo_queue = true + name = "${local.prefix}-events-dlq.fifo" + fifo_queue = true visibility_timeout_seconds = 360 - + redrive_policy = jsonencode({ deadLetterTargetArn = aws_sqs_queue.fifo_resync_dlq_queue.arn maxReceiveCount = 1