From d47a98f80271ce9f3a28243e6076a8c168583537 Mon Sep 17 00:00:00 2001
From: David Byron <dbyron@salesforce.com>
Date: Fri, 10 Sep 2021 10:40:48 -0700
Subject: [PATCH] feat(instrumentation): Gate - Echo Events Metrics and Logging

Wire up metrics around before and after manual pipeline execution events sent to Echo.
Enable debug logging of specific event ids sent to Echo.
---
 .../gate/services/PipelineService.groovy      | 48 ++++++++++++++++++-
 1 file changed, 47 insertions(+), 1 deletion(-)

diff --git a/gate-web/src/main/groovy/com/netflix/spinnaker/gate/services/PipelineService.groovy b/gate-web/src/main/groovy/com/netflix/spinnaker/gate/services/PipelineService.groovy
index 41060526a3..d3232c31c3 100644
--- a/gate-web/src/main/groovy/com/netflix/spinnaker/gate/services/PipelineService.groovy
+++ b/gate-web/src/main/groovy/com/netflix/spinnaker/gate/services/PipelineService.groovy
@@ -16,6 +16,9 @@
 
 package com.netflix.spinnaker.gate.services
 
+import com.netflix.spectator.api.Registry
+import com.netflix.spectator.api.histogram.PercentileTimer
+import com.netflix.spectator.api.patterns.IntervalCounter;
 import com.netflix.spinnaker.gate.services.internal.EchoService
 import com.netflix.spinnaker.gate.services.internal.Front50Service
 import com.netflix.spinnaker.gate.services.internal.OrcaServiceSelector
@@ -27,6 +30,9 @@ import groovy.util.logging.Slf4j
 import org.springframework.beans.factory.annotation.Autowired
 import org.springframework.stereotype.Component
 
+import javax.annotation.PostConstruct
+import java.util.concurrent.TimeUnit
+
 @Component
 @Slf4j
 class PipelineService {
@@ -48,6 +54,27 @@ class PipelineService {
 
   private final RetrySupport retrySupport = new RetrySupport()
 
+  @Autowired
+  Registry registry
+
+  // Echo Event Metrics
+  private IntervalCounter echoEventsIntervalCounter;
+  private PercentileTimer echoEventsPercentileTimer;
+  private IntervalCounter echoEventsErrorIntervalCounter;
+
+  @PostConstruct
+  public void postConstruct() {
+    // Metrics for Echo Event handling.
+    final String idPrefix = "echo.events";
+
+    this.echoEventsIntervalCounter =
+      IntervalCounter.get(this.registry, this.registry.createId(idPrefix + ".count"));
+    this.echoEventsPercentileTimer =
+      PercentileTimer.get(this.registry, this.registry.createId(idPrefix + ".duration"));
+    this.echoEventsErrorIntervalCounter =
+      IntervalCounter.get(this.registry, this.registry.createId(idPrefix + ".error"));
+  }
+
   void deleteForApplication(String applicationName, String pipelineName) {
     front50Service.deletePipelineConfig(applicationName, pipelineName)
   }
@@ -112,7 +139,26 @@ class PipelineService {
       ],
       eventId: eventId
     ]
-    echoService.postEvent(eventMap)
+
+    final long startTimeNanos = registry.clock().monotonicTime();
+
+    try {
+      echoService.postEvent(eventMap)
+    } catch (Exception e) {
+      echoEventsErrorIntervalCounter.increment();
+      log.error("Event processing failure: eventId={}, event={}", eventId, eventMap, e);
+      throw(e)
+    }
+
+    // Echo Event Metrics
+    final long durationInNanos = registry.clock().monotonicTime() - startTimeNanos;
+    echoEventsIntervalCounter.increment();
+    echoEventsPercentileTimer.record(durationInNanos, TimeUnit.NANOSECONDS);
+
+    log.debug(
+      "Event processing success: durationInNanos={}, eventId={}",
+      durationInNanos, eventId);
+
     return [
       eventId: eventId,
       ref    : String.format("/pipelines/%s", executionId)