Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

PLT 290 : Added “uri” projection as part of method level metrics #2423

Merged
merged 3 commits into from
Oct 27, 2023
Merged

PLT 290 : Added “uri” projection as part of method level metrics #2423

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f7dd6f5
Merge pull request #2407 from atlanhq/batch-error
sumandas0 Oct 25, 2023
1c6e298
Merge pull request #2420 from atlanhq/nb/hasLineage
nikhilbonte21 Oct 25, 2023
f755d99
PLT-290: Add “uri” as one of the projections for method level metrics
n5nk Oct 27, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 5 additions & 7 deletions common/src/main/java/org/apache/atlas/service/metrics/MetricUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import io.micrometer.prometheus.PrometheusConfig;
import io.micrometer.prometheus.PrometheusMeterRegistry;
import org.apache.atlas.ApplicationProperties;
import org.apache.atlas.AtlasException;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Expand Down Expand Up @@ -36,18 +37,15 @@ public class MetricUtils {
private static final PrometheusMeterRegistry METER_REGISTRY;

static {
METER_REGISTRY = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT);
METER_REGISTRY.config().withHighCardinalityTagsDetector().commonTags(SERVICE, ATLAS_METASTORE, INTEGRATION, LOCAL);
Metrics.globalRegistry.add(METER_REGISTRY);
}

public MetricUtils() {
try {
METRIC_URI_PATTERNS_MAP = Arrays.stream(ApplicationProperties.get().getStringArray(ATLAS_METRICS_URI_PATTERNS))
.distinct().collect(Collectors.toMap(uri->uri, uri->uri.replaceAll(REGEX_URI_PLACEHOLDER, "*")));
} catch (Exception e) {
LOG.error("Failed to load 'atlas.metrics.uri_patterns from properties");
}
METER_REGISTRY = new PrometheusMeterRegistry(PrometheusConfig.DEFAULT);
METER_REGISTRY.config().withHighCardinalityTagsDetector().commonTags(SERVICE, ATLAS_METASTORE, INTEGRATION, LOCAL);
Metrics.globalRegistry.add(METER_REGISTRY);
}

public Timer.Sample start(String uri) {
Expand Down Expand Up @@ -78,7 +76,7 @@ private Tags getTags(String httpMethod, int httpResponseStatus, String uri) {
URI, matchCanonicalPattern(uri).get());
}

private Optional<String> matchCanonicalPattern(String uri) {
public static Optional<String> matchCanonicalPattern(String uri) {
if (Objects.isNull(uri) || uri.isEmpty()) {
return Optional.empty();
}
Expand Down
2 changes: 1 addition & 1 deletion common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistry.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@

public interface MetricsRegistry {

void collect(String requestId, AtlasPerfMetrics metrics);
void collect(String requestId, String requestUri, AtlasPerfMetrics metrics);

void scrape(PrintWriter writer) throws IOException;

Expand Down
5 changes: 3 additions & 2 deletions common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistryServiceImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ public class MetricsRegistryServiceImpl implements MetricsRegistry {
private static final Logger LOG = LoggerFactory.getLogger(MetricsRegistryServiceImpl.class);

private static final String NAME = "name";
private static final String URI = "uri";
private static final String METHOD_DIST_SUMMARY = "method_dist_summary";
private static final double[] PERCENTILES = {0.99};
private static final String METHOD_LEVEL_METRICS_ENABLE = "atlas.metrics.method_level.enable";
Expand All @@ -40,7 +41,7 @@ public MetricsRegistryServiceImpl() throws AtlasException {
}

@Override
public void collect(String requestId, AtlasPerfMetrics metrics) {
public void collect(String requestId, String requestUri, AtlasPerfMetrics metrics) {
try {
if (!ApplicationProperties.get().getBoolean(METHOD_LEVEL_METRICS_ENABLE, false)) {
return;
Expand All @@ -49,7 +50,7 @@ public void collect(String requestId, AtlasPerfMetrics metrics) {
for (String name : this.filteredMethods) {
if(metrics.hasMetric(name)) {
AtlasPerfMetrics.Metric metric = metrics.getMetric(name);
Timer.builder(METHOD_DIST_SUMMARY).tags(Tags.of(NAME, metric.getName())).publishPercentiles(PERCENTILES)
Timer.builder(METHOD_DIST_SUMMARY).tags(Tags.of(NAME, metric.getName(), URI, requestUri)).publishPercentiles(PERCENTILES)
.register(getMeterRegistry()).record(metric.getTotalTimeMSecs(), TimeUnit.MILLISECONDS);
}
}
Expand Down
3 changes: 2 additions & 1 deletion intg/src/main/java/org/apache/atlas/AtlasErrorCode.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -269,7 +269,8 @@ public enum AtlasErrorCode {
FAILED_TO_REFRESH_TYPE_DEF_CACHE(500, "ATLAS-500-00-20", "Failed to refresh type-def cache"),
CINV_UNHEALTHY(500, "ATLAS-500-00-21", "Unable to process type-definition operations"),
RUNTIME_EXCEPTION(500, "ATLAS-500-00-020", "Runtime exception {0}"),
KEYCLOAK_INIT_FAILED(500, "ATLAS-500-00-021", "Failed to initialize keycloak client: {0}"),
KEYCLOAK_INIT_FAILED(500, "ATLAS-500-00-022", "Failed to initialize keycloak client: {0}"),
BATCH_SIZE_TOO_LARGE(406, "ATLAS-406-00-001", "Batch size is too large, please use a smaller batch size"),


CLASSIFICATION_CURRENTLY_BEING_PROPAGATED(400, "ATLAS-400-00-105", "Classification {0} is currently being propagated."),
Expand Down
22 changes: 21 additions & 1 deletion repository/src/main/java/org/apache/atlas/GraphTransactionInterceptor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,11 @@ public Object invoke(MethodInvocation invocation) throws Throwable {
} else {
doRollback(logRollback, t);
}
throw t;
if (checkForBatchTooLargeError(t)) {
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I guess these changes have come from the previous PR.

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

yes.

throw new AtlasBaseException(AtlasErrorCode.BATCH_SIZE_TOO_LARGE, t);
} else {
throw t;
}
}
} finally {
RequestContext.get().endMetricRecord(metric);
Expand Down Expand Up @@ -167,6 +171,22 @@ public Object invoke(MethodInvocation invocation) throws Throwable {
}
}

public boolean checkForBatchTooLargeError(Throwable t) {
Throwable currentCause = t;
while (currentCause != null) {
String message = currentCause.getMessage();
if (message != null &&
message.contains("Batch too large") &&
currentCause.getClass().getSimpleName().equals("InvalidQueryException")) {
return true;
}
currentCause = currentCause.getCause();
}
return false;
}



private void doCommitOrRollback(final String invokingClass, final String invokedMethodName) {
if (innerFailure.get()) {
if (LOG.isDebugEnabled()) {
Expand Down
12 changes: 10 additions & 2 deletions server-api/src/main/java/org/apache/atlas/RequestContext.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ public class RequestContext {
private MetricsRegistry metricsRegistry;
private boolean skipAuthorizationCheck = false;
private Set<String> deletedEdgesIdsForResetHasLineage = new HashSet<>(0);

private String requestUri;

private RequestContext() {
}
Expand Down Expand Up @@ -159,7 +159,7 @@ public void clearCache() {
if (metrics != null && !metrics.isEmpty()) {
METRICS.debug(metrics.toString());
if (Objects.nonNull(this.metricsRegistry)){
this.metricsRegistry.collect(traceId, metrics);
this.metricsRegistry.collect(traceId, this.requestUri, metrics);
}
metrics.clear();
}
Expand Down Expand Up @@ -642,6 +642,14 @@ public void setMetricRegistry(MetricsRegistry metricsRegistry) {
this.metricsRegistry = metricsRegistry;
}

public void setUri(String uri) {
this.requestUri = uri;
}

public String getRequestUri() {
return this.requestUri;
}

public class EntityGuidPair {
private final Object entity;
private final String guid;
Expand Down
2 changes: 2 additions & 0 deletions webapp/src/main/java/org/apache/atlas/web/filters/AuditFilter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@

import org.apache.atlas.*;
import org.apache.atlas.authorize.AtlasAuthorizationUtils;
import org.apache.atlas.service.metrics.MetricUtils;
import org.apache.atlas.service.metrics.MetricsRegistry;
import org.apache.atlas.util.AtlasRepositoryConfiguration;
import org.apache.atlas.web.util.DateTimeHelper;
Expand Down Expand Up @@ -95,6 +96,7 @@ public void doFilter(ServletRequest request, ServletResponse response, FilterCha

RequestContext.clear();
RequestContext requestContext = RequestContext.get();
requestContext.setUri(MetricUtils.matchCanonicalPattern(httpRequest.getRequestURI()).orElse(EMPTY));
requestContext.setTraceId(internalRequestId);
requestContext.setUser(user, userGroups);
requestContext.setClientIPAddress(AtlasAuthorizationUtils.getRequestIpAddress(httpRequest));
Expand Down
Loading