From ca85707d8ea2bb3d46aaeb3106216bc31c1afdd4 Mon Sep 17 00:00:00 2001
From: Chad Horohoe <chad.horohoe@radai.com>
Date: Wed, 6 Nov 2024 02:55:49 -0800
Subject: [PATCH] feat(lambda-promtail): allow prefix matching for CloudWatch
 log groups

If you have a lot of log groups with similar names this is much easier than listing them all explicitly. Or the bonus feature of just including them all by specifying a prefix of "".

While we're here, introduce the ability to add filter patterns on the subscription to narrow the matched logs as desired.
---
 tools/lambda-promtail/main.tf      | 28 +++++++++++++++++++++++-----
 tools/lambda-promtail/variables.tf | 12 ++++++++++++
 2 files changed, 35 insertions(+), 5 deletions(-)

diff --git a/tools/lambda-promtail/main.tf b/tools/lambda-promtail/main.tf
index 11c97513798d..a7ef7b76b687 100644
--- a/tools/lambda-promtail/main.tf
+++ b/tools/lambda-promtail/main.tf
@@ -245,9 +245,26 @@ resource "aws_lambda_permission" "lambda_promtail_allow_cloudwatch" {
   principal     = "logs.${data.aws_region.current.name}.amazonaws.com"
 }
 
-# This block allows for easily subscribing to multiple log groups via the `log_group_names` var.
-# However, if you need to provide an actual filter_pattern for a specific log group you should
-# copy this block and modify it accordingly.
+# Providing a log group prefix of "" enables matching _all_ log groups
+data "aws_cloudwatch_log_groups" "lambdafunction_logs" {
+  for_each = var.log_group_prefixes
+
+  log_group_name_prefix = each.value
+}
+
+locals {
+  # Combine prefix-generated names from the data source call with the explicitly defined
+  # names to get the full set of log groups to create subscription filters for.
+  # Be sure to remove "/aws/lambda/${var.name}" so we don't log ourselves into oblivion.
+  log_group_names = setsubtract(
+    setunion(
+      var.log_group_names,
+      data.aws_cloudwatch_log_groups.lambdafunction_logs.log_group_names
+    ),
+    toset(["/aws/lambda/${var.name}"])
+  )
+}
+
 resource "aws_cloudwatch_log_subscription_filter" "lambdafunction_logfilter" {
   for_each = var.log_group_names
 
@@ -255,8 +272,9 @@ resource "aws_cloudwatch_log_subscription_filter" "lambdafunction_logfilter" {
   log_group_name  = each.value
   destination_arn = aws_lambda_function.this.arn
 
-  # required but can be empty string
-  filter_pattern = ""
+  # Default to no filter at all (empty string), but allow callers to narrow their
+  # search as desired.
+  filter_pattern = lookup(var.log_group_subscription_filter_patterns, each.value, "")
 }
 
 #-------------------------------------------------------------------------------
diff --git a/tools/lambda-promtail/variables.tf b/tools/lambda-promtail/variables.tf
index 74dceb1a4a19..103db5645e46 100644
--- a/tools/lambda-promtail/variables.tf
+++ b/tools/lambda-promtail/variables.tf
@@ -28,6 +28,18 @@ variable "log_group_names" {
   default     = []
 }
 
+variable "log_group_prefixes" {
+  type        = set(string)
+  description = "List of prefixes to match CloudWatch Log Group names to create Subscription Filters for."
+  default     = []
+}
+
+variable "log_group_subscription_filter_patterns" {
+  type        = map(string)
+  description = "Filter subscription patterns. The key is the log group and the value is the filter pattern."
+  default     = {}
+}
+
 variable "lambda_promtail_image" {
   type        = string
   description = "The ECR image URI to pull and use for lambda-promtail."