From 70d13d5c7efca5be72ddf6344994bf047a182117 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Thu, 18 Jan 2024 17:13:32 +0530 Subject: [PATCH] Split to avoid max_clause_limit issue in ES DSL query --- .../atlas/authorizer/AccessorsExtractor.java | 17 ------------ .../atlas/authorizer/AuthorizerUtils.java | 3 ++- .../authorizers/EntityAuthorizer.java | 26 +++++++++++++++---- .../authorizers/ListAuthorizer.java | 17 +++++++++++- 4 files changed, 39 insertions(+), 24 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/authorizer/AccessorsExtractor.java b/repository/src/main/java/org/apache/atlas/authorizer/AccessorsExtractor.java index b8956b8250..96cb62b61b 100644 --- a/repository/src/main/java/org/apache/atlas/authorizer/AccessorsExtractor.java +++ b/repository/src/main/java/org/apache/atlas/authorizer/AccessorsExtractor.java @@ -64,25 +64,8 @@ public class AccessorsExtractor { public static AtlasAccessorResponse getAccessors(AtlasAccessorRequest request) throws AtlasBaseException { return getAccessorsInMemory(request); - //return getAccessorsES(request); } - public static AtlasAccessorResponse getAccessorsES(AtlasAccessorRequest request) throws AtlasBaseException { - AtlasAccessorResponse response = new AtlasAccessorResponse(); - - String action = AtlasPrivilege.valueOf(request.getAction()).getType(); - - List resourcePolicies = PoliciesStore.getRelevantPolicies(null, null, "atlas", Arrays.asList(action), POLICY_TYPE_ALLOW, true); - resourcePolicies.addAll(PoliciesStore.getRelevantPolicies(null, null, "atlas_tag", Arrays.asList(action), POLICY_TYPE_ALLOW, true)); - - - List abacPolicies = PoliciesStore.getRelevantPolicies(null, null, "atlas_abac", Arrays.asList(action), POLICY_TYPE_ALLOW, true); - - - return response; - } - - private static void collectSubjects(AtlasAccessorResponse response, List matchedPolicies) { for (RangerPolicy policy: matchedPolicies) { List policyItems = null; diff --git a/repository/src/main/java/org/apache/atlas/authorizer/AuthorizerUtils.java b/repository/src/main/java/org/apache/atlas/authorizer/AuthorizerUtils.java index 63cd0d42a7..6494de4be7 100644 --- a/repository/src/main/java/org/apache/atlas/authorizer/AuthorizerUtils.java +++ b/repository/src/main/java/org/apache/atlas/authorizer/AuthorizerUtils.java @@ -42,6 +42,7 @@ public class AuthorizerUtils { public static final String POLICY_TYPE_ALLOW = "allow"; public static final String POLICY_TYPE_DENY = "deny"; + public static final int MAX_CLAUSE_LIMIT = 1024; private static AtlasTypeRegistry typeRegistry; private static EntityGraphRetriever entityRetriever; @@ -237,7 +238,7 @@ public static Map getPreFilterDsl(String persona, String purpos return ListAuthorizer.getElasticsearchDSL(persona, purpose, actions); } - private T getResourceAsObject(String resourceName, Class clazz) throws IOException { + private T getResourceAsObject(String resourceName, Class clazz) throws IOException { InputStream stream = getClass().getResourceAsStream(resourceName); return AtlasType.fromJson(stream, clazz); } diff --git a/repository/src/main/java/org/apache/atlas/authorizer/authorizers/EntityAuthorizer.java b/repository/src/main/java/org/apache/atlas/authorizer/authorizers/EntityAuthorizer.java index 457a08c92b..b9fa38941d 100644 --- a/repository/src/main/java/org/apache/atlas/authorizer/authorizers/EntityAuthorizer.java +++ b/repository/src/main/java/org/apache/atlas/authorizer/authorizers/EntityAuthorizer.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Lists; import org.apache.atlas.RequestContext; import org.apache.atlas.authorize.AtlasAuthorizationUtils; import org.apache.atlas.authorizer.AccessResult; @@ -15,6 +16,7 @@ import org.apache.atlas.repository.graphdb.janus.AtlasElasticsearchQuery; import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; import org.apache.atlas.utils.AtlasPerfMetrics; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.elasticsearch.client.RestClient; import org.slf4j.Logger; @@ -23,6 +25,7 @@ import java.util.*; import java.util.stream.Collectors; +import static org.apache.atlas.authorizer.AuthorizerUtils.MAX_CLAUSE_LIMIT; import static org.apache.atlas.authorizer.AuthorizerUtils.POLICY_TYPE_ALLOW; import static org.apache.atlas.authorizer.AuthorizerUtils.POLICY_TYPE_DENY; import static org.apache.atlas.authorizer.authorizers.AuthorizerCommon.getMap; @@ -400,8 +403,8 @@ public static AccessResult isAccessAllowedEvaluator(String entityTypeName, Strin public static Map getElasticsearchDSL(String persona, String purpose, List actions) { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("EntityAuthorizer.getElasticsearchDSL"); - Map allowDsl = getElasticsearchDSLForPolicyType(persona, purpose, actions, POLICY_TYPE_ALLOW); - Map denyDsl = getElasticsearchDSLForPolicyType(persona, purpose, actions, POLICY_TYPE_DENY); + Map allowDsl = ListAuthorizer.getElasticsearchDSLForPolicyType(persona, purpose, actions, POLICY_TYPE_ALLOW); + Map denyDsl = ListAuthorizer.getElasticsearchDSLForPolicyType(persona, purpose, actions, POLICY_TYPE_DENY); Map finaDsl = new HashMap<>(); if (allowDsl != null) { finaDsl.put("filter", allowDsl); @@ -427,7 +430,7 @@ private static Integer getCountFromElasticsearch(String query) throws AtlasBaseE return count; } - public static Map getElasticsearchDSLForPolicyType(String persona, String purpose, List actions, String policyType) { + /*public static Map getElasticsearchDSLForPolicyType(String persona, String purpose, List actions, String policyType) { List resourcePolicies = PoliciesStore.getRelevantPolicies(persona, purpose, "atlas", actions, policyType); List> resourcePoliciesClauses = ListAuthorizer.getDSLForResourcePolicies(resourcePolicies); @@ -453,11 +456,24 @@ public static Map getElasticsearchDSLForPolicyType(String person } } else { - boolClause.put("should", shouldClauses); + //boolClause.put("should", shouldClauses); + if (shouldClauses.size() > MAX_CLAUSE_LIMIT) { + List> splittedShould = new ArrayList<>(); + List>> partitionedShouldClause = Lists.partition(shouldClauses, MAX_CLAUSE_LIMIT); + + for (List> chunk : partitionedShouldClause) { + splittedShould.add(getMap("bool", getMap("should", chunk))); + } + boolClause.put("should", splittedShould); + + } else { + boolClause.put("should", shouldClauses); + } + boolClause.put("minimum_should_match", 1); } return getMap("bool", boolClause); - } + }*/ } diff --git a/repository/src/main/java/org/apache/atlas/authorizer/authorizers/ListAuthorizer.java b/repository/src/main/java/org/apache/atlas/authorizer/authorizers/ListAuthorizer.java index a39a79245a..2f7f63ac70 100644 --- a/repository/src/main/java/org/apache/atlas/authorizer/authorizers/ListAuthorizer.java +++ b/repository/src/main/java/org/apache/atlas/authorizer/authorizers/ListAuthorizer.java @@ -3,6 +3,7 @@ import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.collect.Lists; import org.apache.atlas.RequestContext; import org.apache.atlas.authorize.AtlasAuthorizationUtils; import org.apache.atlas.authorizer.JsonToElasticsearchQuery; @@ -17,6 +18,7 @@ import java.util.*; import java.util.stream.Collectors; +import static org.apache.atlas.authorizer.AuthorizerUtils.MAX_CLAUSE_LIMIT; import static org.apache.atlas.authorizer.AuthorizerUtils.POLICY_TYPE_ALLOW; import static org.apache.atlas.authorizer.AuthorizerUtils.POLICY_TYPE_DENY; import static org.apache.atlas.authorizer.authorizers.AuthorizerCommon.*; @@ -70,7 +72,20 @@ public static Map getElasticsearchDSLForPolicyType(String person } } else { - boolClause.put("should", shouldClauses); + //boolClause.put("should", shouldClauses); + if (shouldClauses.size() > MAX_CLAUSE_LIMIT) { + List> splittedShould = new ArrayList<>(); + List>> partitionedShouldClause = Lists.partition(shouldClauses, MAX_CLAUSE_LIMIT); + + for (List> chunk : partitionedShouldClause) { + splittedShould.add(getMap("bool", getMap("should", chunk))); + } + boolClause.put("should", splittedShould); + + } else { + boolClause.put("should", shouldClauses); + } + boolClause.put("minimum_should_match", 1); }