Fix missing IAM permissions for Fluent Bit

The service account for Fluent Bit currently does not have permission to create the log streams with the new log stream template name. This means that non-system logs won't be sent to Cloudwatch. This updates the IAM policy to allow creating these log groups, set retention policies, and put log events within the Flightdeck namespace.
thoughtbot · Dec 18, 2023 · 2e8e346 · 2e8e346
1 parent 98bd2d2
commit 2e8e346
Show file tree

Hide file tree

Showing 4 changed files with 39 additions and 25 deletions.
diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml
@@ -43,6 +43,7 @@ jobs:
       run: |
         CLUSTER=$(echo 'flightdeck-${{ github.ref_name }}' | cut -c1-20)
         CLUSTER="$CLUSTER-sandbox-v1"
+        echo "CLUSTER=$CLUSTER" >> "$GITHUB_ENV"
         aws \
           --region us-east-1 \
           eks \
@@ -64,4 +65,6 @@ jobs:
 
     - name: Run tests
       run: |
-        make tests ADDRESS=https://${{ github.ref_name }}.flightdeck-test.thoughtbot.com
+        make tests \
+          ADDRESS=https://${{ github.ref_name }}.flightdeck-test.thoughtbot.com \
+          CLUSTER="$CLUSTER"
diff --git a/aws/platform/main.tf b/aws/platform/main.tf
@@ -370,6 +370,7 @@ locals {
         [OUTPUT]
             Name cloudwatch_logs
             Match *
+            auto_create_group true
             region ${data.aws_region.current.name}
             log_group_name ${module.cloudwatch_logs.log_group_name}
             log_group_template ${var.logs_prefix}/$kubernetes['namespace_name']

diff --git a/aws/platform/modules/cloudwatch-logs/main.tf b/aws/platform/modules/cloudwatch-logs/main.tf
@@ -27,36 +27,16 @@ resource "aws_iam_role_policy_attachment" "this" {
 
 data "aws_iam_policy_document" "this" {
   statement {
-    sid = "AllowCreateLogEvents"
+    sid = "AllowCreateLogs"
     actions = [
-      "logs:DescribeLogStreams",
-      "logs:PutLogEvents"
-    ]
-    resources = [
-      "${aws_cloudwatch_log_group.this.arn}:log-stream:*"
-    ]
-  }
-
-  statement {
-    sid = "AllowCreateLogGroup"
-    actions = [
-      "logs:CreateLogGroup"
+      "logs:CreateLogGroup",
+      "logs:CreateLogStream",
+      "logs:PutRetentionPolicy",
     ]
     resources = [
       "${local.arn_prefix}:log-group:${var.log_group_prefix}/*"
     ]
   }
-
-  statement {
-    sid = "AllowCreateLogStream"
-    actions = [
-      "logs:CreateLogStream"
-    ]
-    resources = [
-      aws_cloudwatch_log_group.this.arn,
-      "${aws_cloudwatch_log_group.this.arn}:log-stream:*"
-    ]
-  }
 }
 
 data "aws_caller_identity" "current" {}

diff --git a/tests/fluentbit.bats b/tests/fluentbit.bats
@@ -29,3 +29,33 @@
     false
   fi
 }
+
+@test "creates log streams within groups for Kubernetes namespaces" {
+  expected="$RANDOM"
+  curl -v "$ADDRESS/echo?log=$expected"
+  pod=$(kubectl \
+    get pod \
+    --field-selector=status.phase=Running \
+    --selector=app=echoserver \
+    -n acceptance \
+    --output=name \
+    | cut -d'/' -f2)
+  logs=$(aws \
+    --region us-east-1 \
+    logs \
+    get-log-events \
+    --log-group-name "/flightdeck/acceptance" \
+    --log-stream-name "$pod.echoserver" \
+    --query 'events[*].[message]' \
+      --output text)
+
+  if ! echo "$logs" | grep -q "log=$expected"; then
+    echo "Failed to find log for test request." >&2
+    echo >&2
+    echo "Test request was: GET /echo?log=$expected" >&2
+    echo >&2
+    echo "Found log entries" >&2
+    echo "$logs" >&2
+    false
+  fi
+}