Create MSK Serverless scrape config

Netflix-Skunkworks · Dec 10, 2024 · 9bddb7b · 9bddb7b
1 parent 45c3303
commit 9bddb7b
Show file tree

Hide file tree

Showing 2 changed files with 119 additions and 0 deletions.
diff --git a/atlas-cloudwatch/src/main/resources/msk-serverless.conf b/atlas-cloudwatch/src/main/resources/msk-serverless.conf
@@ -0,0 +1,104 @@
+atlas {
+  cloudwatch {
+
+    // https://docs.aws.amazon.com/msk/latest/developerguide/serverless-monitoring.html
+    msk-serverless-cluster = {
+      namespace = "AWS/Kafka"
+      period = 1m
+      end-period-offset = 5
+
+      dimensions = [
+        "Cluster Name",
+        "Topic"
+      ]
+
+      metrics = [
+        {
+          name = "BytesIn"
+          alias = "aws.mskserverless.bytes"
+          conversion = "sum,rate"
+          tags = [
+            {
+              key = "id"
+              value = "in"
+            }
+          ]
+        },
+        {
+          name = "BytesOut"
+          alias = "aws.mskserverless.bytes"
+          conversion = "sum,rate"
+          tags = [
+            {
+              key = "id"
+              value = "out"
+            }
+          ]
+        },
+        {
+          name = "FetchMessageConversions"
+          alias = "aws.mskserverless.messageConversions"
+          conversion = "sum,rate"
+          tags = [
+            {
+              key = "id"
+              value = "fetch"
+            }
+          ]
+        },
+        {
+          name = "ProduceMessageConversions"
+          alias = "aws.mskserverless.messageConversions"
+          conversion = "sum,rate"
+          tags = [
+            {
+              key = "id"
+              value = "produce"
+            }
+          ]
+        },
+        {
+          name = "MessagesIn"
+          alias = "aws.mskserverless.messages"
+          conversion = "sum,rate"
+          tags = [
+            {
+              key = "id"
+              value = "in"
+            }
+          ]
+        }
+      ]
+    }
+
+    msk-serverless-consumergroup = {
+      namespace = "AWS/Kafka"
+      period = 1m
+      end-period-offset = 5
+
+      dimensions = [
+        "Cluster Name",
+        "Topic",
+        "Consumer Group"
+      ]
+
+      metrics = [
+        {
+          name = "EstimatedMaxTimeLag"
+          alias = "aws.mskserverless.estimatedMaxTimeLag"
+          conversion = "max"
+        },
+        {
+          name = "MaxOffsetLag"
+          alias = "aws.mskserverless.maxOffsetLag"
+          conversion = "max"
+        },
+        {
+          name = "SumOffsetLag"
+          alias = "aws.mskserverless.sumOffsetLag"
+          conversion = "max"
+        }
+      ]
+    }
+  }
+}
diff --git a/atlas-cloudwatch/src/main/resources/reference.conf b/atlas-cloudwatch/src/main/resources/reference.conf
@@ -302,6 +302,14 @@ atlas {
           name = "ClusterName"
           alias = "aws.memorydb"
         },
+        {
+          name = "Cluster Name"
+          alias = "aws.cluster"
+        },
+        {
+          name = "Consumer Group"
+          alias = "aws.consumerGroup"
+        },
         {
           name = "ConnectionId"
           alias = "aws.connection"
@@ -474,6 +482,10 @@ atlas {
           name = "TableName"
           alias = "aws.table"
         },
+        {
+          name = "Topic"
+          alias = "aws.topic"
+        },
         {
           name = "TopicName"
           alias = "aws.topic"
@@ -574,6 +586,8 @@ atlas {
       "medialive-output",
       "medialive-network",
       "memorydb",
+      "msk-serverless-cluster",
+      "msk-serverless-consumergroup",
       "nat-gateway",
       "neptune-cluster-role",
       "neptune-cluster-5m",
@@ -627,6 +641,7 @@ include "lambda.conf"
 include "mediaconnect.conf"
 include "medialive.conf"
 include "memorydb.conf"
+include "msk-serverless.conf"
 include "nat-gateway.conf"
 include "neptune.conf"
 include "nlb.conf"