diff --git a/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistry.java b/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistry.java index 1730fb50de..a1b2d1e6eb 100644 --- a/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistry.java +++ b/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistry.java @@ -4,11 +4,14 @@ import java.io.IOException; import java.io.PrintWriter; +import java.util.List; public interface MetricsRegistry { void collect(String requestId, String requestUri, AtlasPerfMetrics metrics); + void collect(String requestId, String requestUri, List applicationMetrics); + void scrape(PrintWriter writer) throws IOException; } diff --git a/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistryServiceImpl.java b/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistryServiceImpl.java index 65a5be31f9..660f1b2160 100644 --- a/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistryServiceImpl.java +++ b/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistryServiceImpl.java @@ -1,6 +1,7 @@ package org.apache.atlas.service.metrics; import io.micrometer.core.instrument.Metrics; +import io.micrometer.core.instrument.Tag; import io.micrometer.core.instrument.Tags; import io.micrometer.core.instrument.Timer; import io.micrometer.prometheus.PrometheusMeterRegistry; @@ -16,6 +17,7 @@ import java.io.PrintWriter; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -30,6 +32,7 @@ public class MetricsRegistryServiceImpl implements MetricsRegistry { private static final String NAME = "name"; private static final String URI = "uri"; private static final String METHOD_DIST_SUMMARY = "method_dist_summary"; + private static final String APPLICATION_LEVEL_METRICS_SUMMARY = "application_level_metrics_summary"; private static final double[] PERCENTILES = {0.99}; private static final String METHOD_LEVEL_METRICS_ENABLE = "atlas.metrics.method_level.enable"; private static final String ATLAS_METRICS_METHOD_PATTERNS = "atlas.metrics.method_patterns"; @@ -59,6 +62,23 @@ public void collect(String requestId, String requestUri, AtlasPerfMetrics metric return; } } + public void collect(String requestId, String requestUri, List applicationMetrics){ + try { + for(AtlasPerfMetrics.Metric metric : applicationMetrics){ + Timer.builder(APPLICATION_LEVEL_METRICS_SUMMARY).tags(convertToMicrometerTags(metric.getTags())).publishPercentiles(PERCENTILES) + .register(getMeterRegistry()).record(metric.getTotalTimeMSecs(), TimeUnit.MILLISECONDS); + } + } catch (Exception e) { + LOG.error("Failed to collect metrics", e); + return; + } + } + + private static Iterable convertToMicrometerTags(Map tagsMap) { + return tagsMap.entrySet().stream() + .map(entry -> Tag.of(entry.getKey(), entry.getValue())) + .collect(Collectors.toList()); + } @Override public void scrape(PrintWriter writer) { diff --git a/common/src/main/java/org/apache/atlas/utils/AtlasPerfMetrics.java b/common/src/main/java/org/apache/atlas/utils/AtlasPerfMetrics.java index 382afb04db..beebcb6ab1 100644 --- a/common/src/main/java/org/apache/atlas/utils/AtlasPerfMetrics.java +++ b/common/src/main/java/org/apache/atlas/utils/AtlasPerfMetrics.java @@ -19,6 +19,7 @@ package org.apache.atlas.utils; +import java.util.HashMap; import java.util.LinkedHashMap; import java.util.Map; import java.util.Set; @@ -105,6 +106,7 @@ public static class Metric { private final String name; private long invocations = 0; private long totalTimeMSecs = 0; + HashMap tags = new HashMap<>(); public Metric(String name) { this.name = name; @@ -118,8 +120,20 @@ public long getInvocations() { return invocations; } + public void setTotalTimeMSecs(long totalTimeMSecs) { + this.totalTimeMSecs = totalTimeMSecs; + } + public long getTotalTimeMSecs() { return totalTimeMSecs; } + + public void addTag(String key, String value) { + tags.put(key, value); + } + public HashMap getTags() { + return tags; + } + } } diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java b/server-api/src/main/java/org/apache/atlas/RequestContext.java index 9d73318e99..bcc4c0085b 100644 --- a/server-api/src/main/java/org/apache/atlas/RequestContext.java +++ b/server-api/src/main/java/org/apache/atlas/RequestContext.java @@ -58,6 +58,7 @@ public class RequestContext { private final Set processGuidIds = new HashSet<>(); private final AtlasPerfMetrics metrics = isMetricsEnabled ? new AtlasPerfMetrics() : null; + private final List applicationMetrics = new ArrayList<>(); private List entityGuidInRequest = null; private final Set entitiesToSkipUpdate = new HashSet<>(); private final Set onlyCAUpdateEntities = new HashSet<>(); @@ -172,6 +173,12 @@ public void clearCache() { } metrics.clear(); } + if (CollectionUtils.isNotEmpty(applicationMetrics)) { + if (Objects.nonNull(this.metricsRegistry)){ + this.metricsRegistry.collect(traceId, this.requestUri, applicationMetrics); + } + applicationMetrics.clear(); + } setTraceId(null); if (this.entityGuidInRequest != null) { @@ -179,6 +186,10 @@ public void clearCache() { } } + public void addApplicationMetrics(AtlasPerfMetrics.Metric metric) { + this.applicationMetrics.add(metric); + } + public void clearEntityCache() { this.entityCache.clear(); } diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java index 1e2a94f44d..20dd5d9019 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java @@ -38,6 +38,7 @@ import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasStructType; import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.atlas.utils.AtlasPerfTracer; import org.apache.atlas.web.util.Servlets; import org.apache.commons.collections.CollectionUtils; @@ -65,7 +66,6 @@ import java.util.HashSet; import java.util.List; import java.util.Set; - import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; import static org.apache.atlas.repository.Constants.REQUEST_HEADER_HOST; import static org.apache.atlas.repository.Constants.REQUEST_HEADER_USER_AGENT; @@ -389,7 +389,7 @@ public AtlasSearchResult indexSearch(@Context HttpServletRequest servletRequest, RequestContext.get().setIncludeMeanings(!parameters.isExcludeMeanings()); RequestContext.get().setIncludeClassifications(!parameters.isExcludeClassifications()); - try { + try { if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DiscoveryREST.indexSearch(" + parameters + ")"); } @@ -425,6 +425,14 @@ public AtlasSearchResult indexSearch(@Context HttpServletRequest servletRequest, } throw abe; } finally { + if(parameters.getUtmTags() != null) { + AtlasPerfMetrics.Metric indexsearchMetric = new AtlasPerfMetrics.Metric("indexsearch"); + indexsearchMetric.addTag("utmTags", String.join(",", parameters.getUtmTags())); + indexsearchMetric.addTag("name", "indexsearch"); + indexsearchMetric.addTag("querySize",parameters.getDsl().getOrDefault("size", 20).toString()); + indexsearchMetric.setTotalTimeMSecs(System.currentTimeMillis() - startTime); + RequestContext.get().addApplicationMetrics(indexsearchMetric); + } AtlasPerfTracer.log(perf); } }