Merge pull request #2818 from atlanhq/app-metric

Add additional App metrics
atlanhq · Feb 1, 2024 · 923e5db · 923e5db
2 parents 4a277cf + cb48422
commit 923e5db
Show file tree

Hide file tree

Showing 5 changed files with 58 additions and 2 deletions.
diff --git a/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistry.java b/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistry.java
@@ -4,11 +4,14 @@
 
 import java.io.IOException;
 import java.io.PrintWriter;
+import java.util.List;
 
 public interface MetricsRegistry {
 
     void collect(String requestId, String requestUri, AtlasPerfMetrics metrics);
 
+    void collect(String requestId, String requestUri, List<AtlasPerfMetrics.Metric> applicationMetrics);
+
     void scrape(PrintWriter writer) throws IOException;
 
 }
diff --git a/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistryServiceImpl.java b/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistryServiceImpl.java
@@ -1,6 +1,7 @@
 package org.apache.atlas.service.metrics;
 
 import io.micrometer.core.instrument.Metrics;
+import io.micrometer.core.instrument.Tag;
 import io.micrometer.core.instrument.Tags;
 import io.micrometer.core.instrument.Timer;
 import io.micrometer.prometheus.PrometheusMeterRegistry;
@@ -16,6 +17,7 @@
 import java.io.PrintWriter;
 import java.util.Arrays;
 import java.util.List;
+import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
@@ -30,6 +32,7 @@ public class MetricsRegistryServiceImpl implements MetricsRegistry {
     private static final String NAME = "name";
     private static final String URI = "uri";
     private static final String METHOD_DIST_SUMMARY = "method_dist_summary";
+    private static final String APPLICATION_LEVEL_METRICS_SUMMARY = "application_level_metrics_summary";
     private static final double[] PERCENTILES = {0.99};
     private static final String METHOD_LEVEL_METRICS_ENABLE = "atlas.metrics.method_level.enable";
     private static final String ATLAS_METRICS_METHOD_PATTERNS = "atlas.metrics.method_patterns";
@@ -59,6 +62,23 @@ public void collect(String requestId, String requestUri, AtlasPerfMetrics metric
             return;
         }
     }
+    public void collect(String requestId, String requestUri, List<AtlasPerfMetrics.Metric> applicationMetrics){
+        try {
+            for(AtlasPerfMetrics.Metric metric : applicationMetrics){
+                Timer.builder(APPLICATION_LEVEL_METRICS_SUMMARY).tags(convertToMicrometerTags(metric.getTags())).publishPercentiles(PERCENTILES)
+                        .register(getMeterRegistry()).record(metric.getTotalTimeMSecs(), TimeUnit.MILLISECONDS);
+            }
+        } catch (Exception e) {
+            LOG.error("Failed to collect metrics", e);
+            return;
+        }
+    }
+
+    private static Iterable<Tag> convertToMicrometerTags(Map<String, String> tagsMap) {
+        return tagsMap.entrySet().stream()
+                .map(entry -> Tag.of(entry.getKey(), entry.getValue()))
+                .collect(Collectors.toList());
+    }
 
     @Override
     public void scrape(PrintWriter writer) {

diff --git a/common/src/main/java/org/apache/atlas/utils/AtlasPerfMetrics.java b/common/src/main/java/org/apache/atlas/utils/AtlasPerfMetrics.java
@@ -19,6 +19,7 @@
 package org.apache.atlas.utils;
 
 
+import java.util.HashMap;
 import java.util.LinkedHashMap;
 import java.util.Map;
 import java.util.Set;
@@ -105,6 +106,7 @@ public static class Metric {
         private final String name;
         private long invocations = 0;
         private long totalTimeMSecs = 0;
+        HashMap<String, String> tags = new HashMap<>();
 
         public Metric(String name) {
             this.name = name;
@@ -118,8 +120,20 @@ public long getInvocations() {
             return invocations;
         }
 
+        public void setTotalTimeMSecs(long totalTimeMSecs) {
+            this.totalTimeMSecs = totalTimeMSecs;
+        }
+
         public long getTotalTimeMSecs() {
             return totalTimeMSecs;
         }
+
+        public void addTag(String key, String value) {
+            tags.put(key, value);
+        }
+        public HashMap<String, String> getTags() {
+            return tags;
+        }
+
     }
 }
diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java b/server-api/src/main/java/org/apache/atlas/RequestContext.java
@@ -58,6 +58,7 @@ public class RequestContext {
     private final Set<String>                            processGuidIds      = new HashSet<>();
 
     private final AtlasPerfMetrics metrics = isMetricsEnabled ? new AtlasPerfMetrics() : null;
+    private final List<AtlasPerfMetrics.Metric> applicationMetrics = new ArrayList<>();
     private List<EntityGuidPair> entityGuidInRequest = null;
     private final Set<String> entitiesToSkipUpdate = new HashSet<>();
     private final Set<String> onlyCAUpdateEntities = new HashSet<>();
@@ -172,13 +173,23 @@ public void clearCache() {
             }
             metrics.clear();
         }
+        if (CollectionUtils.isNotEmpty(applicationMetrics)) {
+            if (Objects.nonNull(this.metricsRegistry)){
+                this.metricsRegistry.collect(traceId, this.requestUri, applicationMetrics);
+            }
+            applicationMetrics.clear();
+        }
         setTraceId(null);
 
         if (this.entityGuidInRequest != null) {
             this.entityGuidInRequest.clear();
         }
     }
 
+    public void addApplicationMetrics(AtlasPerfMetrics.Metric metric) {
+        this.applicationMetrics.add(metric);
+    }
+
     public void clearEntityCache() {
         this.entityCache.clear();
     }

diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java
@@ -38,6 +38,7 @@
 import org.apache.atlas.type.AtlasEntityType;
 import org.apache.atlas.type.AtlasStructType;
 import org.apache.atlas.type.AtlasTypeRegistry;
+import org.apache.atlas.utils.AtlasPerfMetrics;
 import org.apache.atlas.utils.AtlasPerfTracer;
 import org.apache.atlas.web.util.Servlets;
 import org.apache.commons.collections.CollectionUtils;
@@ -65,7 +66,6 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
-
 import static org.apache.atlas.repository.Constants.QUALIFIED_NAME;
 import static org.apache.atlas.repository.Constants.REQUEST_HEADER_HOST;
 import static org.apache.atlas.repository.Constants.REQUEST_HEADER_USER_AGENT;
@@ -389,7 +389,7 @@ public AtlasSearchResult indexSearch(@Context HttpServletRequest servletRequest,
 
         RequestContext.get().setIncludeMeanings(!parameters.isExcludeMeanings());
         RequestContext.get().setIncludeClassifications(!parameters.isExcludeClassifications());
-        try {
+        try     {
             if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
                 perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DiscoveryREST.indexSearch(" + parameters + ")");
             }
@@ -425,6 +425,14 @@ public AtlasSearchResult indexSearch(@Context HttpServletRequest servletRequest,
             }
             throw abe;
         } finally {
+            if(parameters.getUtmTags() != null) {
+                AtlasPerfMetrics.Metric indexsearchMetric = new AtlasPerfMetrics.Metric("indexsearch");
+                indexsearchMetric.addTag("utmTags", String.join(",", parameters.getUtmTags()));
+                indexsearchMetric.addTag("name", "indexsearch");
+                indexsearchMetric.addTag("querySize",parameters.getDsl().getOrDefault("size", 20).toString());
+                indexsearchMetric.setTotalTimeMSecs(System.currentTimeMillis() - startTime);
+                RequestContext.get().addApplicationMetrics(indexsearchMetric);
+            }
             AtlasPerfTracer.log(perf);
         }
     }