From 55f22b505e5e93e26a123530a6d8adf92d7b4046 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Wed, 3 Jan 2024 15:38:51 +0530 Subject: [PATCH] Refactorings and metric loggings --- .../atlas/authorizer/AuthorizerUtils.java | 38 ++++++++----------- .../atlas/authorizer/EntityAuthorizer.java | 19 ++++++++-- .../JsonToElasticsearchQuery.java | 2 +- .../atlas/authorizer/ListAuthorizer.java | 9 ++++- .../atlas/authorizer/PoliciesStore.java | 16 +++++++- .../authorizer/RelationshipAuthorizer.java | 23 +++++++++-- .../discovery/EntityDiscoveryService.java | 4 +- 7 files changed, 74 insertions(+), 37 deletions(-) rename repository/src/main/java/org/apache/atlas/{discovery => authorizer}/JsonToElasticsearchQuery.java (99%) diff --git a/repository/src/main/java/org/apache/atlas/authorizer/AuthorizerUtils.java b/repository/src/main/java/org/apache/atlas/authorizer/AuthorizerUtils.java index acf1f0e3bd..3fb016c7f6 100644 --- a/repository/src/main/java/org/apache/atlas/authorizer/AuthorizerUtils.java +++ b/repository/src/main/java/org/apache/atlas/authorizer/AuthorizerUtils.java @@ -9,6 +9,7 @@ import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -24,30 +25,8 @@ public class AuthorizerUtils { private static final Logger LOG = LoggerFactory.getLogger(AuthorizerUtils.class); - private EntityDiscoveryService discoveryService; - private static AtlasTypeRegistry typeRegistry; private static AuthorizerUtils authorizerUtils; - public AuthorizerUtils (EntityDiscoveryService discoveryService, AtlasTypeRegistry typeRegistry) { - try { - this.discoveryService = discoveryService; - this.typeRegistry = typeRegistry; - - LOG.info("==> AtlasAuthorization"); - } catch (Exception e) { - LOG.error("==> AtlasAuthorization -> Error!"); - } - } - - public static AuthorizerUtils getInstance(EntityDiscoveryService discoveryService, AtlasTypeRegistry typeRegistry) { - synchronized (AuthorizerUtils.class) { - if (authorizerUtils == null) { - authorizerUtils = new AuthorizerUtils(discoveryService, typeRegistry); - } - return authorizerUtils; - } - } - public static void verifyUpdateEntityAccess(AtlasEntityHeader entityHeader) throws AtlasBaseException { if (!SKIP_UPDATE_AUTH_CHECK_TYPES.contains(entityHeader.getTypeName())) { verifyAccess(entityHeader.getGuid(), AtlasPrivilege.ENTITY_UPDATE.getType()); @@ -61,6 +40,7 @@ public static void verifyDeleteEntityAccess(AtlasEntityHeader entityHeader) thro } public static void verifyEntityCreateAccess(AtlasEntity entity, AtlasPrivilege action) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("verifyEntityCreateAccess"); String userName = AuthorizerCommon.getCurrentUserName(); if (StringUtils.isEmpty(userName) || RequestContext.get().isImportInProgress()) { @@ -76,10 +56,13 @@ public static void verifyEntityCreateAccess(AtlasEntity entity, AtlasPrivilege a } } catch (AtlasBaseException e) { throw e; + } finally { + RequestContext.get().endMetricRecord(recorder); } } public static void verifyAccess(String guid, String action) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("verifyAccess"); String userName = AuthorizerCommon.getCurrentUserName(); if (StringUtils.isEmpty(userName) || RequestContext.get().isImportInProgress()) { @@ -92,10 +75,13 @@ public static void verifyAccess(String guid, String action) throws AtlasBaseExce } } catch (AtlasBaseException e) { throw e; + } finally { + RequestContext.get().endMetricRecord(recorder); } } public static void verifyAccessForEvaluator(String entityTypeName, String entityQualifiedName, String action) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("verifyAccessForEvaluator"); String userName = AuthorizerCommon.getCurrentUserName(); if (StringUtils.isEmpty(userName) || RequestContext.get().isImportInProgress()) { @@ -108,10 +94,13 @@ public static void verifyAccessForEvaluator(String entityTypeName, String entity } } catch (AtlasBaseException e) { throw e; + } finally { + RequestContext.get().endMetricRecord(recorder); } } public static void verifyRelationshipAccess(String action, String endOneGuid, String endTwoGuid) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("verifyRelationshipAccess"); String userName = AuthorizerCommon.getCurrentUserName(); if (StringUtils.isEmpty(userName) || RequestContext.get().isImportInProgress()) { @@ -124,10 +113,13 @@ public static void verifyRelationshipAccess(String action, String endOneGuid, St } } catch (AtlasBaseException e) { throw e; + } finally { + RequestContext.get().endMetricRecord(recorder); } } public static void verifyRelationshipCreateAccess(String action, String relationshipType, AtlasEntityHeader endOneEntity, AtlasEntityHeader endTwoEntity) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("verifyRelationshipCreateAccess"); String userName = AuthorizerCommon.getCurrentUserName(); if (StringUtils.isEmpty(userName) || RequestContext.get().isImportInProgress()) { @@ -141,6 +133,8 @@ public static void verifyRelationshipCreateAccess(String action, String relation } } catch (AtlasBaseException e) { throw e; + } finally { + RequestContext.get().endMetricRecord(recorder); } } diff --git a/repository/src/main/java/org/apache/atlas/authorizer/EntityAuthorizer.java b/repository/src/main/java/org/apache/atlas/authorizer/EntityAuthorizer.java index 225e846dc4..959f3ab3b2 100644 --- a/repository/src/main/java/org/apache/atlas/authorizer/EntityAuthorizer.java +++ b/repository/src/main/java/org/apache/atlas/authorizer/EntityAuthorizer.java @@ -49,6 +49,7 @@ public static boolean isAccessAllowedInMemory(AtlasEntity entity, String action) } public static boolean isAccessAllowedInMemory(AtlasEntity entity, String action, String policyType) { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("isAccessAllowedInMemory."+policyType); List policies = PoliciesStore.getRelevantPolicies(null, null, "atlas_abac", Arrays.asList(action), policyType); List filterCriteriaList = new ArrayList<>(); for (RangerPolicy policy : policies) { @@ -88,10 +89,12 @@ public static boolean isAccessAllowedInMemory(AtlasEntity entity, String action, ret = validateResourcesForCreateEntityInMemory(tagPolicies, entity); } + RequestContext.get().endMetricRecord(recorder); return ret; } private static boolean validateResourcesForCreateEntityInMemory(List resourcePolicies, AtlasEntity entity) { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("validateResourcesForCreateEntityInMemory"); RangerPolicy matchedPolicy = null; Set entityTypes = AuthorizerCommon.getTypeAndSupertypesList(entity.getTypeName()); @@ -186,11 +189,12 @@ private static boolean validateResourcesForCreateEntityInMemory(List assetTypes) { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("evaluateFilterCriteria"); String operator = crit.get("operator").asText(); String attributeName = crit.get("attributeName").asText(); String attributeValue = crit.get("attributeValue").asText(); @@ -345,10 +350,12 @@ private static boolean evaluateFilterCriteria(JsonNode crit, AtlasEntity entity, default: LOG.warn("Found unknown operator {}", operator); } + RequestContext.get().endMetricRecord(recorder); return false; } public static boolean isAccessAllowed(String guid, String action) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("EntityAuthorizer.isAccessAllowed"); if (guid == null) { return false; } @@ -370,6 +377,8 @@ public static boolean isAccessAllowed(String guid, String action) throws AtlasBa if (count != null && count > 0) { return true; } + + RequestContext.get().endMetricRecord(recorder); return false; } @@ -398,6 +407,7 @@ public static boolean isAccessAllowedEvaluator(String entityTypeName, String ent } public static Map getElasticsearchDSL(String persona, String purpose, List actions) { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("EntityAuthorizer.getElasticsearchDSL"); Map allowDsl = getElasticsearchDSLForPolicyType(persona, purpose, actions, POLICY_TYPE_ALLOW); Map denyDsl = getElasticsearchDSLForPolicyType(persona, purpose, actions, POLICY_TYPE_DENY); Map finaDsl = new HashMap<>(); @@ -407,10 +417,12 @@ public static Map getElasticsearchDSL(String persona, String pur if (denyDsl != null) { finaDsl.put("must_not", denyDsl); } + RequestContext.get().endMetricRecord(recorder); return getMap("bool", finaDsl); } private static Integer getCountFromElasticsearch(String query) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("EntityAuthorizer.getCountFromElasticsearch"); RestClient restClient = getLowLevelClient(); AtlasElasticsearchQuery elasticsearchQuery = new AtlasElasticsearchQuery("janusgraph_vertex_index", restClient); Map elasticsearchResult = null; @@ -419,6 +431,7 @@ private static Integer getCountFromElasticsearch(String query) throws AtlasBaseE if (elasticsearchResult!=null) { count = (Integer) elasticsearchResult.get("total"); } + RequestContext.get().endMetricRecord(recorder); return count; } @@ -455,6 +468,4 @@ public static Map getElasticsearchDSLForPolicyType(String person return getMap("bool", boolClause); } - - } diff --git a/repository/src/main/java/org/apache/atlas/discovery/JsonToElasticsearchQuery.java b/repository/src/main/java/org/apache/atlas/authorizer/JsonToElasticsearchQuery.java similarity index 99% rename from repository/src/main/java/org/apache/atlas/discovery/JsonToElasticsearchQuery.java rename to repository/src/main/java/org/apache/atlas/authorizer/JsonToElasticsearchQuery.java index ad39901a04..6ee7f0b363 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/JsonToElasticsearchQuery.java +++ b/repository/src/main/java/org/apache/atlas/authorizer/JsonToElasticsearchQuery.java @@ -1,4 +1,4 @@ -package org.apache.atlas.discovery; +package org.apache.atlas.authorizer; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; diff --git a/repository/src/main/java/org/apache/atlas/authorizer/ListAuthorizer.java b/repository/src/main/java/org/apache/atlas/authorizer/ListAuthorizer.java index 0fc92c578a..4ef922278e 100644 --- a/repository/src/main/java/org/apache/atlas/authorizer/ListAuthorizer.java +++ b/repository/src/main/java/org/apache/atlas/authorizer/ListAuthorizer.java @@ -3,8 +3,9 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; -import org.apache.atlas.discovery.JsonToElasticsearchQuery; +import org.apache.atlas.RequestContext; import org.apache.atlas.plugin.model.RangerPolicy; +import org.apache.atlas.utils.AtlasPerfMetrics; import java.util.*; @@ -13,6 +14,7 @@ public class ListAuthorizer { public static Map getElasticsearchDSL(String persona, String purpose, List actions) { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("ListAuthorizer.getElasticsearchDSL"); Map allowDsl = getElasticsearchDSLForPolicyType(persona, purpose, actions, POLICY_TYPE_ALLOW); Map denyDsl = getElasticsearchDSLForPolicyType(persona, purpose, actions, POLICY_TYPE_DENY); Map finaDsl = new HashMap<>(); @@ -22,10 +24,13 @@ public static Map getElasticsearchDSL(String persona, String pur if (denyDsl != null) { finaDsl.put("must_not", denyDsl); } + + RequestContext.get().endMetricRecord(recorder); return getMap("bool", finaDsl); } public static Map getElasticsearchDSLForPolicyType(String persona, String purpose, List actions, String policyType) { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("ListAuthorizer.getElasticsearchDSLForPolicyType."+ policyType); List resourcePolicies = PoliciesStore.getRelevantPolicies(persona, purpose, "atlas", actions, policyType); List> resourcePoliciesClauses = getDSLForResourcePolicies(resourcePolicies); @@ -55,8 +60,8 @@ public static Map getElasticsearchDSLForPolicyType(String person boolClause.put("minimum_should_match", 1); } + RequestContext.get().endMetricRecord(recorder); return getMap("bool", boolClause); - } public static List> getDSLForResourcePolicies(List policies) { diff --git a/repository/src/main/java/org/apache/atlas/authorizer/PoliciesStore.java b/repository/src/main/java/org/apache/atlas/authorizer/PoliciesStore.java index 33f8ae88b6..f56ad0876b 100644 --- a/repository/src/main/java/org/apache/atlas/authorizer/PoliciesStore.java +++ b/repository/src/main/java/org/apache/atlas/authorizer/PoliciesStore.java @@ -1,8 +1,10 @@ package org.apache.atlas.authorizer; +import org.apache.atlas.RequestContext; import org.apache.atlas.plugin.model.RangerPolicy; import org.apache.atlas.plugin.util.RangerRoles; import org.apache.atlas.plugin.util.RangerUserStore; +import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -54,6 +56,7 @@ private static List getAbacPolicies() { } public static List getRelevantPolicies(String persona, String purpose, String serviceName, List actions, String policyType) { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("getRelevantPolicies"); String policyQualifiedNamePrefix = null; if (persona != null && !persona.isEmpty()) { policyQualifiedNamePrefix = persona; @@ -85,11 +88,13 @@ public static List getRelevantPolicies(String persona, String purp policies = getFilteredPoliciesForUser(policies, user, groups, roles, policyType); policies = getFilteredPoliciesForActions(policies, actions, policyType); } - return policies; + RequestContext.get().endMetricRecord(recorder); + return policies; } static List getFilteredPoliciesForQualifiedName(List policies, String qualifiedNamePrefix) { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("getFilteredPoliciesForQualifiedName"); if (qualifiedNamePrefix != null && !qualifiedNamePrefix.isEmpty()) { List filteredPolicies = new ArrayList<>(); for(RangerPolicy policy : policies) { @@ -99,10 +104,13 @@ static List getFilteredPoliciesForQualifiedName(List } return filteredPolicies; } + + RequestContext.get().endMetricRecord(recorder); return policies; } private static List getFilteredPoliciesForActions(List policies, List actions, String type) { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("getFilteredPoliciesForActions"); List filteredPolicies = new ArrayList<>(); for(RangerPolicy policy : policies) { RangerPolicy.RangerPolicyItem policyItem = null; @@ -123,10 +131,14 @@ private static List getFilteredPoliciesForActions(List getFilteredPoliciesForUser(List policies, String user, List groups, List roles, String type) { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("getFilteredPoliciesForUser"); + List filterPolicies = new ArrayList<>(); for(RangerPolicy policy : policies) { RangerPolicy.RangerPolicyItem policyItem = null; @@ -147,6 +159,8 @@ private static List getFilteredPoliciesForUser(List } } } + + RequestContext.get().endMetricRecord(recorder); return filterPolicies; } } diff --git a/repository/src/main/java/org/apache/atlas/authorizer/RelationshipAuthorizer.java b/repository/src/main/java/org/apache/atlas/authorizer/RelationshipAuthorizer.java index ca2983a4bc..c290ccfd16 100644 --- a/repository/src/main/java/org/apache/atlas/authorizer/RelationshipAuthorizer.java +++ b/repository/src/main/java/org/apache/atlas/authorizer/RelationshipAuthorizer.java @@ -4,7 +4,6 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.atlas.RequestContext; -import org.apache.atlas.discovery.JsonToElasticsearchQuery; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.TypeCategory; import org.apache.atlas.model.glossary.relations.AtlasTermAssignmentHeader; @@ -45,7 +44,7 @@ public static boolean isAccessAllowedInMemory(String action, String relationship public static boolean checkRelationshipAccessAllowedInMemory(String action, String relationshipType, AtlasEntityHeader endOneEntity, AtlasEntityHeader endTwoEntity, String policyType) throws AtlasBaseException { //Relationship add, update, remove access check in memory - AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("isRelationshipAccessAllowed."+policyType); + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("checkRelationshipAccessAllowedInMemory."+policyType); try { List policies = PoliciesStore.getRelevantPolicies(null, null, "atlas_abac", Arrays.asList(action), policyType); @@ -101,6 +100,8 @@ public static boolean checkRelationshipAccessAllowedInMemory(String action, Stri private static boolean validateResourcesForCreateRelationship(List resourcePolicies, String relationshipType, AtlasEntityHeader endOneEntity, AtlasEntityHeader endTwoEntity) { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("validateResourcesForCreateRelationship"); + RangerPolicy matchedPolicy = null; Set endOneEntityTypes = AuthorizerCommon.getTypeAndSupertypesList(endOneEntity.getTypeName()); @@ -242,11 +243,13 @@ private static boolean validateResourcesForCreateRelationship(List } } + RequestContext.get().endMetricRecord(recorder); return false; } public static boolean validateFilterCriteriaWithEntity(JsonNode data, AtlasEntity entity) { - AtlasPerfMetrics.MetricRecorder convertJsonToQueryMetrics = RequestContext.get().startMetricRecord("convertJsonToQuery"); + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("RelationshipAuthorizer.validateFilterCriteriaWithEntity"); + String condition = data.get("condition").asText(); JsonNode criterion = data.get("criterion"); @@ -360,11 +363,13 @@ public static boolean validateFilterCriteriaWithEntity(JsonNode data, AtlasEntit } } - RequestContext.get().endMetricRecord(convertJsonToQueryMetrics); + RequestContext.get().endMetricRecord(recorder); return result; } public static boolean isRelationshipAccessAllowed(String action, String endOneGuid, String endTwoGuid) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("RelationshipAuthorizer.isRelationshipAccessAllowed"); + //Relationship update, remove access check with ES query if (endOneGuid == null || endTwoGuid == null) { return false; @@ -417,11 +422,15 @@ public static boolean isRelationshipAccessAllowed(String action, String endOneGu LOG.info(dslString); } catch (JsonProcessingException e) { return false; + } finally { + RequestContext.get().endMetricRecord(recorder); } return false; } public static Map getElasticsearchDSLForRelationshipActions(List actions, String endOneGuid, String endTwoGuid) throws JsonProcessingException { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("RelationshipAuthorizer.getElasticsearchDSLForRelationshipActions"); + List> policiesClauses = new ArrayList<>(); List resourcePolicies = PoliciesStore.getRelevantPolicies(null, null, "atlas", actions, POLICY_TYPE_ALLOW); List> resourcePoliciesClauses = getDSLForRelationshipResourcePolicies(resourcePolicies); @@ -458,10 +467,14 @@ public static Map getElasticsearchDSLForRelationshipActions(List Map boolClause = new HashMap<>(); boolClause.put("filter", clauses); + + RequestContext.get().endMetricRecord(recorder); return getMap("query", getMap("bool", boolClause)); } private static List> getDSLForRelationshipResourcePolicies(List policies) { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("RelationshipAuthorizer.getDSLForRelationshipResourcePolicies"); + List> shouldClauses = new ArrayList<>(); for (RangerPolicy policy : policies) { if (!policy.getResources().isEmpty() && "RELATIONSHIP".equals(policy.getPolicyResourceCategory())) { @@ -491,6 +504,8 @@ private static List> getDSLForRelationshipResourcePolicies(L } } } + + RequestContext.get().endMetricRecord(recorder); return shouldClauses; } diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index 2eeec71e79..116bae0ba0 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -107,7 +107,6 @@ public class EntityDiscoveryService implements AtlasDiscoveryService { private final SuggestionsProvider suggestionsProvider; private final DSLQueryExecutor dslQueryExecutor; private final StatsClient statsClient; - private final AuthorizerUtils authorizerUtils; @Inject public EntityDiscoveryService(AtlasTypeRegistry typeRegistry, @@ -132,7 +131,6 @@ public EntityDiscoveryService(AtlasTypeRegistry typeRegistry, this.dslQueryExecutor = AtlasConfiguration.DSL_EXECUTOR_TRAVERSAL.getBoolean() ? new TraversalBasedExecutor(typeRegistry, graph, entityRetriever) : new ScriptEngineBasedExecutor(typeRegistry, graph, entityRetriever); - this.authorizerUtils = AuthorizerUtils.getInstance(this, typeRegistry); } @Override @@ -1035,7 +1033,7 @@ private void addPreFiltersToSearchQuery(SearchParams searchParams) { List actions = new ArrayList<>(); actions.add("entity-read"); - Map allPreFiltersBoolClause = authorizerUtils.getPreFilterDsl(persona, purpose, actions); + Map allPreFiltersBoolClause = AuthorizerUtils.getPreFilterDsl(persona, purpose, actions); mustClauseList.add(allPreFiltersBoolClause); String dslString = searchParams.getQuery();