Add the concept of a higher threshold alarm metric

For example, we don't care about every instance of "email already taken" from Identity, but we want to know if there are suddenly lots of them.
guardian · Jan 30, 2025 · 4e053a4 · 4e053a4
1 parent 181564f
commit 4e053a4
Show file tree

Hide file tree

Showing 2 changed files with 31 additions and 6 deletions.
diff --git a/support-frontend/app/actions/CustomActionBuilders.scala b/support-frontend/app/actions/CustomActionBuilders.scala
@@ -62,28 +62,45 @@ class CustomActionBuilders(
 
   case class LoggingAndAlarmOnFailure[A](chainedAction: Action[A]) extends EssentialAction with SafeLogging {
 
+    private def pushMetric(cloudwatchEvent: AwsCloudWatchMetricPut.MetricRequest) = {
+      AwsCloudWatchMetricPut(AwsCloudWatchMetricPut.client)(cloudwatchEvent)
+    }
     private def pushAlarmMetric = {
       val cloudwatchEvent = AwsCloudWatchMetricSetup.serverSideCreateFailure(stage)
-      AwsCloudWatchMetricPut(AwsCloudWatchMetricPut.client)(cloudwatchEvent)
+      pushMetric(cloudwatchEvent)
+    }
+
+    private def pushHighThresholdAlarmMetric = {
+      val cloudwatchEvent = AwsCloudWatchMetricSetup.serverSideHighThresholdCreateFailure(stage)
+      pushMetric(cloudwatchEvent)
     }
 
     private def maybePushAlarmMetric(result: Result) = {
+      // We'll never alarm on these
       val ignoreList = Set(
         emailProviderRejectedCode,
         invalidEmailAddressCode,
         recaptchaFailedCode,
+      )
+      // We'll alarm on these, but only over a certain threshold
+      val highThresholdList = Set(
         emailAddressAlreadyTakenCode,
       )
       if (result.header.status == 500) {
-        if (!ignoreList.contains(result.header.reasonPhrase.getOrElse(""))) {
+        if (ignoreList.contains(result.header.reasonPhrase.getOrElse(""))) {
+          logger.info(
+            s"not pushing alarm metric for ${result.header.status} ${result.header.reasonPhrase} as it is in our ignore list",
+          )
+        } else if (highThresholdList.contains(result.header.reasonPhrase.getOrElse(""))) {
+          logger.info(
+            s"pushing higher threshold alarm metric for ${result.header.status} ${result.header.reasonPhrase}",
+          )
+          pushHighThresholdAlarmMetric
+        } else {
           logger.error(
             scrub"pushing alarm metric - non 2xx response. Http code: ${result.header.status}, reason: ${result.header.reasonPhrase}",
           )
           pushAlarmMetric
-        } else {
-          logger.info(
-            s"not pushing alarm metric for ${result.header.status} ${result.header.reasonPhrase} as it is in our ignore list",
-          )
         }
       }
     }

diff --git a/support-services/src/main/scala/com/gu/aws/AwsCloudWatchMetricSetup.scala b/support-services/src/main/scala/com/gu/aws/AwsCloudWatchMetricSetup.scala
@@ -47,6 +47,14 @@ object AwsCloudWatchMetricSetup {
       ),
     )
 
+  def serverSideHighThresholdCreateFailure(stage: Stage): MetricRequest =
+    getMetricRequest(
+      MetricName("ServerSideHighThresholdCreateFailure"),
+      Map(
+        MetricDimensionName("Stage") -> MetricDimensionValue(stage.toString),
+      ),
+    )
+
   def defaultPromotionsLoadingFailure(stage: Stage): MetricRequest =
     getMetricRequest(
       MetricName("DefaultPromotionsLoadingFailure"),