Add CloudWatch log support to AWS Pipeline (closes #7)

snowplow · Aug 13, 2021 · af49f65 · af49f65
1 parent bf2a1d7
commit af49f65
Show file tree

Hide file tree

Showing 6 changed files with 102 additions and 28 deletions.
diff --git a/terraform/aws/pipeline/default/main.tf b/terraform/aws/pipeline/default/main.tf
@@ -12,7 +12,7 @@ locals {
 
 module "s3_pipeline_bucket" {
   source  = "snowplow-devops/s3-bucket/aws"
-  version = "0.1.0"
+  version = "0.1.1"
 
   count = var.s3_bucket_deploy ? 1 : 0
 
@@ -30,7 +30,7 @@ resource "aws_key_pair" "pipeline" {
 # 1. Deploy Kinesis streams
 module "raw_stream" {
   source  = "snowplow-devops/kinesis-stream/aws"
-  version = "0.1.0"
+  version = "0.1.1"
 
   name = "${var.prefix}-raw-stream"
 
@@ -39,7 +39,7 @@ module "raw_stream" {
 
 module "bad_1_stream" {
   source  = "snowplow-devops/kinesis-stream/aws"
-  version = "0.1.0"
+  version = "0.1.1"
 
   name = "${var.prefix}-bad-1-stream"
 
@@ -48,7 +48,7 @@ module "bad_1_stream" {
 
 module "enriched_stream" {
   source  = "snowplow-devops/kinesis-stream/aws"
-  version = "0.1.0"
+  version = "0.1.1"
 
   name = "${var.prefix}-enriched-stream"
 
@@ -57,7 +57,7 @@ module "enriched_stream" {
 
 module "bad_2_stream" {
   source  = "snowplow-devops/kinesis-stream/aws"
-  version = "0.1.0"
+  version = "0.1.1"
 
   name = "${var.prefix}-bad-2-stream"
 
@@ -67,7 +67,7 @@ module "bad_2_stream" {
 # 2. Deploy Collector stack
 module "collector_lb" {
   source  = "snowplow-devops/alb/aws"
-  version = "0.1.1"
+  version = "0.1.2"
 
   name              = "${var.prefix}-collector-lb"
   vpc_id            = var.vpc_id
@@ -82,7 +82,7 @@ module "collector_lb" {
 
 module "collector_kinesis" {
   source  = "snowplow-devops/collector-kinesis-ec2/aws"
-  version = "0.1.1"
+  version = "0.1.2"
 
   name               = "${var.prefix}-collector-server"
   vpc_id             = var.vpc_id
@@ -102,12 +102,15 @@ module "collector_kinesis" {
   iam_permissions_boundary = var.iam_permissions_boundary
 
   tags = var.tags
+
+  cloudwatch_logs_enabled        = var.cloudwatch_logs_enabled
+  cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days
 }
 
 # 3. Deploy Enrichment
 module "enrich_kinesis" {
   source  = "snowplow-devops/enrich-kinesis-ec2/aws"
-  version = "0.1.3"
+  version = "0.1.4"
 
   name                 = "${var.prefix}-enrich-server"
   vpc_id               = var.vpc_id
@@ -130,12 +133,15 @@ module "enrich_kinesis" {
   kcl_write_max_capacity = var.pipeline_kcl_write_max_capacity
 
   tags = var.tags
+
+  cloudwatch_logs_enabled        = var.cloudwatch_logs_enabled
+  cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days
 }
 
 # 4. Deploy Postgres Loader
 module "pipeline_rds" {
   source  = "snowplow-devops/rds/aws"
-  version = "0.1.3"
+  version = "0.1.4"
 
   name        = "${var.prefix}-pipeline-rds"
   vpc_id      = var.vpc_id
@@ -152,7 +158,7 @@ module "pipeline_rds" {
 
 module "postgres_loader_enriched" {
   source  = "snowplow-devops/postgres-loader-kinesis-ec2/aws"
-  version = "0.1.0"
+  version = "0.1.1"
 
   name       = "${var.prefix}-postgres-loader-enriched-server"
   vpc_id     = var.vpc_id
@@ -183,11 +189,14 @@ module "postgres_loader_enriched" {
   kcl_write_max_capacity = var.pipeline_kcl_write_max_capacity
 
   tags = var.tags
+
+  cloudwatch_logs_enabled        = var.cloudwatch_logs_enabled
+  cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days
 }
 
 module "postgres_loader_bad" {
   source  = "snowplow-devops/postgres-loader-kinesis-ec2/aws"
-  version = "0.1.0"
+  version = "0.1.1"
 
   name       = "${var.prefix}-postgres-loader-bad-server"
   vpc_id     = var.vpc_id
@@ -218,12 +227,15 @@ module "postgres_loader_bad" {
   kcl_write_max_capacity = var.pipeline_kcl_write_max_capacity
 
   tags = var.tags
+
+  cloudwatch_logs_enabled        = var.cloudwatch_logs_enabled
+  cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days
 }
 
 # 5. Save raw, enriched and bad data to Amazon S3
 module "s3_loader_raw" {
   source  = "snowplow-devops/s3-loader-kinesis-ec2/aws"
-  version = "0.1.2"
+  version = "0.1.3"
 
   name             = "${var.prefix}-s3-loader-raw-server"
   vpc_id           = var.vpc_id
@@ -244,11 +256,14 @@ module "s3_loader_raw" {
   kcl_write_max_capacity = var.pipeline_kcl_write_max_capacity
 
   tags = var.tags
+
+  cloudwatch_logs_enabled        = var.cloudwatch_logs_enabled
+  cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days
 }
 
 module "s3_loader_bad" {
   source  = "snowplow-devops/s3-loader-kinesis-ec2/aws"
-  version = "0.1.2"
+  version = "0.1.3"
 
   name             = "${var.prefix}-s3-loader-bad-server"
   vpc_id           = var.vpc_id
@@ -269,11 +284,14 @@ module "s3_loader_bad" {
   kcl_write_max_capacity = var.pipeline_kcl_write_max_capacity
 
   tags = var.tags
+
+  cloudwatch_logs_enabled        = var.cloudwatch_logs_enabled
+  cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days
 }
 
 module "s3_loader_enriched" {
   source  = "snowplow-devops/s3-loader-kinesis-ec2/aws"
-  version = "0.1.2"
+  version = "0.1.3"
 
   name             = "${var.prefix}-s3-loader-enriched-server"
   vpc_id           = var.vpc_id
@@ -294,4 +312,7 @@ module "s3_loader_enriched" {
   kcl_write_max_capacity = var.pipeline_kcl_write_max_capacity
 
   tags = var.tags
+
+  cloudwatch_logs_enabled        = var.cloudwatch_logs_enabled
+  cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days
 }
diff --git a/terraform/aws/pipeline/default/terraform.tfvars b/terraform/aws/pipeline/default/terraform.tfvars
@@ -71,3 +71,7 @@ ssl_information = {
 
 # --- Extra Tags to append to created resources (optional)
 tags = {}
+
+# --- CloudWatch logging to ensure logs are saved outside of the server
+cloudwatch_logs_enabled        = true
+cloudwatch_logs_retention_days = 7
diff --git a/terraform/aws/pipeline/default/variables.tf b/terraform/aws/pipeline/default/variables.tf
@@ -120,3 +120,15 @@ variable "tags" {
   default     = {}
   type        = map(string)
 }
+
+variable "cloudwatch_logs_enabled" {
+  description = "Whether application logs should be reported to CloudWatch; by default they are only on the server"
+  default     = false
+  type        = bool
+}
+
+variable "cloudwatch_logs_retention_days" {
+  description = "The length of time in days to retain logs for"
+  default     = 7
+  type        = number
+}
diff --git a/terraform/aws/pipeline/secure/main.tf b/terraform/aws/pipeline/secure/main.tf
@@ -12,7 +12,7 @@ locals {
 
 module "s3_pipeline_bucket" {
   source  = "snowplow-devops/s3-bucket/aws"
-  version = "0.1.0"
+  version = "0.1.1"
 
   count = var.s3_bucket_deploy ? 1 : 0
 
@@ -30,7 +30,7 @@ resource "aws_key_pair" "pipeline" {
 # 1. Deploy Kinesis streams
 module "raw_stream" {
   source  = "snowplow-devops/kinesis-stream/aws"
-  version = "0.1.0"
+  version = "0.1.1"
 
   name = "${var.prefix}-raw-stream"
 
@@ -39,7 +39,7 @@ module "raw_stream" {
 
 module "bad_1_stream" {
   source  = "snowplow-devops/kinesis-stream/aws"
-  version = "0.1.0"
+  version = "0.1.1"
 
   name = "${var.prefix}-bad-1-stream"
 
@@ -48,7 +48,7 @@ module "bad_1_stream" {
 
 module "enriched_stream" {
   source  = "snowplow-devops/kinesis-stream/aws"
-  version = "0.1.0"
+  version = "0.1.1"
 
   name = "${var.prefix}-enriched-stream"
 
@@ -57,7 +57,7 @@ module "enriched_stream" {
 
 module "bad_2_stream" {
   source  = "snowplow-devops/kinesis-stream/aws"
-  version = "0.1.0"
+  version = "0.1.1"
 
   name = "${var.prefix}-bad-2-stream"
 
@@ -67,7 +67,7 @@ module "bad_2_stream" {
 # 2. Deploy Collector stack
 module "collector_lb" {
   source  = "snowplow-devops/alb/aws"
-  version = "0.1.1"
+  version = "0.1.2"
 
   name              = "${var.prefix}-collector-lb"
   vpc_id            = var.vpc_id
@@ -82,7 +82,7 @@ module "collector_lb" {
 
 module "collector_kinesis" {
   source  = "snowplow-devops/collector-kinesis-ec2/aws"
-  version = "0.1.1"
+  version = "0.1.2"
 
   name               = "${var.prefix}-collector-server"
   vpc_id             = var.vpc_id
@@ -104,12 +104,15 @@ module "collector_kinesis" {
   associate_public_ip_address = false
 
   tags = var.tags
+
+  cloudwatch_logs_enabled        = var.cloudwatch_logs_enabled
+  cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days
 }
 
 # 3. Deploy Enrichment
 module "enrich_kinesis" {
   source  = "snowplow-devops/enrich-kinesis-ec2/aws"
-  version = "0.1.3"
+  version = "0.1.4"
 
   name                 = "${var.prefix}-enrich-server"
   vpc_id               = var.vpc_id
@@ -134,12 +137,15 @@ module "enrich_kinesis" {
   associate_public_ip_address = false
 
   tags = var.tags
+
+  cloudwatch_logs_enabled        = var.cloudwatch_logs_enabled
+  cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days
 }
 
 # 4. Deploy Postgres Loader
 module "pipeline_rds" {
   source  = "snowplow-devops/rds/aws"
-  version = "0.1.3"
+  version = "0.1.4"
 
   name        = "${var.prefix}-pipeline-rds"
   vpc_id      = var.vpc_id
@@ -156,7 +162,7 @@ module "pipeline_rds" {
 
 module "postgres_loader_enriched" {
   source  = "snowplow-devops/postgres-loader-kinesis-ec2/aws"
-  version = "0.1.0"
+  version = "0.1.1"
 
   name       = "${var.prefix}-postgres-loader-enriched-server"
   vpc_id     = var.vpc_id
@@ -189,11 +195,14 @@ module "postgres_loader_enriched" {
   associate_public_ip_address = false
 
   tags = var.tags
+
+  cloudwatch_logs_enabled        = var.cloudwatch_logs_enabled
+  cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days
 }
 
 module "postgres_loader_bad" {
   source  = "snowplow-devops/postgres-loader-kinesis-ec2/aws"
-  version = "0.1.0"
+  version = "0.1.1"
 
   name       = "${var.prefix}-postgres-loader-bad-server"
   vpc_id     = var.vpc_id
@@ -226,12 +235,15 @@ module "postgres_loader_bad" {
   associate_public_ip_address = false
 
   tags = var.tags
+
+  cloudwatch_logs_enabled        = var.cloudwatch_logs_enabled
+  cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days
 }
 
 # 5. Save raw, enriched and bad data to Amazon S3
 module "s3_loader_raw" {
   source  = "snowplow-devops/s3-loader-kinesis-ec2/aws"
-  version = "0.1.2"
+  version = "0.1.3"
 
   name             = "${var.prefix}-s3-loader-raw-server"
   vpc_id           = var.vpc_id
@@ -254,11 +266,14 @@ module "s3_loader_raw" {
   associate_public_ip_address = false
 
   tags = var.tags
+
+  cloudwatch_logs_enabled        = var.cloudwatch_logs_enabled
+  cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days
 }
 
 module "s3_loader_bad" {
   source  = "snowplow-devops/s3-loader-kinesis-ec2/aws"
-  version = "0.1.2"
+  version = "0.1.3"
 
   name             = "${var.prefix}-s3-loader-bad-server"
   vpc_id           = var.vpc_id
@@ -281,11 +296,14 @@ module "s3_loader_bad" {
   associate_public_ip_address = false
 
   tags = var.tags
+
+  cloudwatch_logs_enabled        = var.cloudwatch_logs_enabled
+  cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days
 }
 
 module "s3_loader_enriched" {
   source  = "snowplow-devops/s3-loader-kinesis-ec2/aws"
-  version = "0.1.2"
+  version = "0.1.3"
 
   name             = "${var.prefix}-s3-loader-enriched-server"
   vpc_id           = var.vpc_id
@@ -308,4 +326,7 @@ module "s3_loader_enriched" {
   associate_public_ip_address = false
 
   tags = var.tags
+
+  cloudwatch_logs_enabled        = var.cloudwatch_logs_enabled
+  cloudwatch_logs_retention_days = var.cloudwatch_logs_retention_days
 }
diff --git a/terraform/aws/pipeline/secure/terraform.tfvars b/terraform/aws/pipeline/secure/terraform.tfvars
@@ -72,3 +72,7 @@ ssl_information = {
 
 # --- Extra Tags to append to created resources (optional)
 tags = {}
+
+# --- CloudWatch logging to ensure logs are saved outside of the server
+cloudwatch_logs_enabled        = true
+cloudwatch_logs_retention_days = 7