From b84869817d95d8a7bb5b2574fe95a31ffe7244e2 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Wed, 21 Aug 2024 18:22:46 +0530 Subject: [PATCH 01/31] Merge pull request #3395 from atlanhq/ns/DG-1709-refresh-delta-policies DG-1709 | Delta based policies refresh --- .../atlas/plugin/model/RangerPolicyDelta.java | 4 + .../policyengine/RangerPolicyRepository.java | 34 +-- .../plugin/service/RangerBasePlugin.java | 1 + .../atlas/plugin/util/PolicyRefresher.java | 38 ++- .../plugin/util/RangerPolicyDeltaUtil.java | 12 +- .../atlas/plugin/util/ServicePolicies.java | 2 +- .../CachePolicyTransformerImpl.java | 237 ++++++++++++++++-- .../apache/atlas/ApplicationProperties.java | 1 + .../org/apache/atlas/web/rest/AuthREST.java | 40 ++- 9 files changed, 299 insertions(+), 70 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java index adfe24af1d..def6c6c0dd 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java @@ -73,6 +73,9 @@ public RangerPolicyDelta(final Long id, final Integer changeType, final Long pol @JsonIgnore public Long getPolicyId() { return policy != null ? policy.getId() : null; } + @JsonIgnore + public String getPolicyGuid() { return policy != null ? policy.getGuid() : null; } + @JsonIgnore public String getZoneName() { return policy != null ? policy.getZoneName() : null; } @@ -94,6 +97,7 @@ public String toString() { + ", serviceType:" + getServiceType() + ", policyType:" + getPolicyType() + ", policyId:[" + getPolicyId() + "]" + + ", policyGuid:[" + getPolicyGuid() + "]" + ", policy:[" + policy +"]"; } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyRepository.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyRepository.java index 5b3b469fb0..4324ccb2cd 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyRepository.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyRepository.java @@ -90,7 +90,7 @@ enum AuditModeEnum { private List dataMaskPolicyEvaluators; private List rowFilterPolicyEvaluators; private final List auditPolicyEvaluators; - private Map policyEvaluatorsMap; + private Map policyEvaluatorsMap; private boolean isContextEnrichersShared = false; private boolean isPreCleaned = false; @@ -654,9 +654,9 @@ public List getLikelyMatchPolicyEvaluators(RangerAccessRe } - public Map getPolicyEvaluatorsMap() { return policyEvaluatorsMap; } + public Map getPolicyEvaluatorsMap() { return policyEvaluatorsMap; } - RangerPolicyEvaluator getPolicyEvaluator(Long id) { + RangerPolicyEvaluator getPolicyEvaluator(String id) { return policyEvaluatorsMap.get(id); } @@ -1252,17 +1252,17 @@ private void removeEvaluatorFromTrie(RangerPolicyEvaluator oldEvaluator, RangerR } } - private Map createPolicyEvaluatorsMap() { - Map tmpPolicyEvaluatorMap = new HashMap<>(); + private Map createPolicyEvaluatorsMap() { + Map tmpPolicyEvaluatorMap = new HashMap<>(); for (RangerPolicyEvaluator evaluator : getPolicyEvaluators()) { - tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator); + tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getGuid(), evaluator); } for (RangerPolicyEvaluator evaluator : getDataMaskPolicyEvaluators()) { - tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator); + tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getGuid(), evaluator); } for (RangerPolicyEvaluator evaluator : getRowFilterPolicyEvaluators()) { - tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator); + tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getGuid(), evaluator); } return tmpPolicyEvaluatorMap; @@ -1294,7 +1294,7 @@ private RangerPolicyEvaluator addPolicy(RangerPolicy policy) { } if (!RangerPolicy.POLICY_TYPE_AUDIT.equals(policy.getPolicyType())) { - policyEvaluatorsMap.put(policy.getId(), ret); + policyEvaluatorsMap.put(policy.getGuid(), ret); } } } @@ -1306,22 +1306,22 @@ private RangerPolicyEvaluator addPolicy(RangerPolicy policy) { return ret; } - private void removePolicy(Long id) { + private void removePolicy(String guid) { if (LOG.isDebugEnabled()) { - LOG.debug("==> RangerPolicyRepository.removePolicy(" + id +")"); + LOG.debug("==> RangerPolicyRepository.removePolicy(" + guid +")"); } Iterator iterator = policies.iterator(); while (iterator.hasNext()) { - if (id.equals(iterator.next().getId())) { + if (guid.equals(iterator.next().getGuid())) { iterator.remove(); //break; } } - policyEvaluatorsMap.remove(id); + policyEvaluatorsMap.remove(guid); if (LOG.isDebugEnabled()) { - LOG.debug("<== RangerPolicyRepository.removePolicy(" + id +")"); + LOG.debug("<== RangerPolicyRepository.removePolicy(" + guid +")"); } } @@ -1355,13 +1355,13 @@ private void deletePolicyEvaluator(RangerPolicyEvaluator evaluator) { } private RangerPolicyEvaluator update(final RangerPolicyDelta delta, final RangerPolicyEvaluator currentEvaluator) { - + LOG.info("PolicyDelta: RangerPolicyRepository.update is called, policyGuid: "+delta.getPolicyGuid()); if (LOG.isDebugEnabled()) { LOG.debug("==> RangerPolicyRepository.update(delta=" + delta + ", currentEvaluator=" + (currentEvaluator == null ? null : currentEvaluator.getPolicy()) + ")"); } Integer changeType = delta.getChangeType(); String policyType = delta.getPolicyType(); - Long policyId = delta.getPolicyId(); + String policyId = delta.getPolicyGuid(); RangerPolicy policy = delta.getPolicy(); @@ -1472,7 +1472,7 @@ private void updateResourceTrie(List deltas) { for (RangerPolicyDelta delta : deltas) { final Integer changeType = delta.getChangeType(); final String serviceType = delta.getServiceType(); - final Long policyId = delta.getPolicyId(); + final String policyId = delta.getPolicyGuid(); final String policyType = delta.getPolicyType(); if (!serviceType.equals(this.serviceDef.getName())) { diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java index b224cccc7e..fb44584e44 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java @@ -379,6 +379,7 @@ public void setPolicies(ServicePolicies policies) { RangerPolicyEngineImpl oldPolicyEngineImpl = (RangerPolicyEngineImpl) oldPolicyEngine; newPolicyEngine = RangerPolicyEngineImpl.getPolicyEngine(oldPolicyEngineImpl, policies); + //TODO: this looks like a mistake, second arg should be servicePolicies which has the applied delta } if (newPolicyEngine != null) { diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java index aae09a7d26..b18906dde4 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -21,8 +21,12 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasException; import org.apache.atlas.authz.admin.client.AtlasAuthAdminClient; import org.apache.atlas.policytransformer.CachePolicyTransformerImpl; +import org.apache.atlas.repository.audit.ESBasedAuditRepository; +import org.apache.commons.configuration.Configuration; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -40,6 +44,8 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; +import static org.apache.atlas.ApplicationProperties.DELTA_BASED_REFRESH; + public class PolicyRefresher extends Thread { private static final Log LOG = LogFactory.getLog(PolicyRefresher.class); @@ -64,6 +70,9 @@ public class PolicyRefresher extends Thread { private long lastActivationTimeInMillis; private boolean policiesSetInPlugin; private boolean serviceDefSetInPlugin; + private Configuration atlasConfig; + private boolean enableDeltaBasedRefresh; + private ESBasedAuditRepository auditRepository; public PolicyRefresher(RangerBasePlugin plugIn) { @@ -104,6 +113,16 @@ public PolicyRefresher(RangerBasePlugin plugIn) { this.userStoreProvider = new RangerUserStoreProvider(getServiceType(), appId, getServiceName(), atlasAuthAdminClient, cacheDir, pluginConfig); this.pollingIntervalMs = pluginConfig.getLong(propertyPrefix + ".policy.pollIntervalMs", 30 * 1000); + try { + this.atlasConfig = ApplicationProperties.get(); + this.auditRepository = new ESBasedAuditRepository(atlasConfig); + this.auditRepository.start(); + this.enableDeltaBasedRefresh = this.atlasConfig.getBoolean(DELTA_BASED_REFRESH, false); + } catch (AtlasException e) { + LOG.error("PolicyDelta: Error while reading atlas configuration", e); + this.enableDeltaBasedRefresh = false; + } + setName("PolicyRefresher(serviceName=" + serviceName + ")-" + getId()); if(LOG.isDebugEnabled()) { @@ -316,13 +335,20 @@ private ServicePolicies loadPolicyfromPolicyAdmin() throws RangerServiceNotFound try { - if (serviceName.equals("atlas") && plugIn.getTypeRegistry() != null && lastUpdatedTiemInMillis == -1) { + if (serviceName.equals("atlas") && plugIn.getTypeRegistry() != null) { RangerRESTUtils restUtils = new RangerRESTUtils(); - CachePolicyTransformerImpl transformer = new CachePolicyTransformerImpl(plugIn.getTypeRegistry()); - - svcPolicies = transformer.getPolicies(serviceName, - restUtils.getPluginId(serviceName, plugIn.getAppId()), - lastUpdatedTiemInMillis); + CachePolicyTransformerImpl transformer = new CachePolicyTransformerImpl(plugIn.getTypeRegistry(), auditRepository); + if (lastUpdatedTiemInMillis == -1) { + svcPolicies = transformer.getPoliciesAll(serviceName, + restUtils.getPluginId(serviceName, plugIn.getAppId()), + lastUpdatedTiemInMillis, null); + } else if (this.enableDeltaBasedRefresh) { + svcPolicies = transformer.getPoliciesDelta(serviceName, + restUtils.getPluginId(serviceName, plugIn.getAppId()), + lastUpdatedTiemInMillis); + } else { + svcPolicies = atlasAuthAdminClient.getServicePoliciesIfUpdated(lastUpdatedTiemInMillis); + } } else { svcPolicies = atlasAuthAdminClient.getServicePoliciesIfUpdated(lastUpdatedTiemInMillis); } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java index e3cb0f3697..b92ae004d4 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java @@ -79,7 +79,7 @@ public static List applyDeltas(List policies, List applyDeltas(List policies, List applyDeltas(List policies, List deltas, String compo for (RangerPolicyDelta delta : deltas) { final Integer changeType = delta.getChangeType(); - final Long policyId = delta.getPolicyId(); + final String policyGuid = delta.getPolicyGuid(); if (changeType == null) { isValid = false; @@ -171,7 +167,7 @@ public static boolean isValidDeltas(List deltas, String compo && changeType != RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE && changeType != RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE) { isValid = false; - } else if (policyId == null) { + } else if (policyGuid == null) { isValid = false; } else { final String serviceType = delta.getServiceType(); diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java index 547349c8f5..1a4dd81f29 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java @@ -390,7 +390,7 @@ static public TagPolicies copyHeader(TagPolicies source, String componentService return ret; } - public static ServicePolicies applyDelta(final ServicePolicies servicePolicies, RangerPolicyEngineImpl policyEngine) { + public static ServicePolicies applyDelta(final ServicePolicies servicePolicies, RangerPolicyEngineImpl policyEngine) { ServicePolicies ret = copyHeader(servicePolicies); List oldResourcePolicies = policyEngine.getResourcePolicies(); diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 4c6b9621fa..f092a488a6 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -18,14 +18,19 @@ package org.apache.atlas.policytransformer; +import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.AtlasException; import org.apache.atlas.RequestContext; import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.audit.AuditSearchParams; +import org.apache.atlas.model.audit.EntityAuditEventV2; +import org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2; +import org.apache.atlas.model.audit.EntityAuditSearchResult; import org.apache.atlas.model.discovery.AtlasSearchResult; import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.instance.AtlasEntityHeader; -import org.apache.atlas.model.instance.AtlasStruct; +import org.apache.atlas.plugin.model.RangerPolicyDelta; import org.apache.atlas.plugin.util.ServicePolicies; import org.apache.atlas.plugin.model.RangerPolicy; import org.apache.atlas.plugin.model.RangerPolicy.RangerDataMaskPolicyItem; @@ -37,6 +42,7 @@ import org.apache.atlas.plugin.model.RangerServiceDef; import org.apache.atlas.plugin.model.RangerValiditySchedule; import org.apache.atlas.plugin.util.ServicePolicies.TagPolicies; +import org.apache.atlas.repository.audit.ESBasedAuditRepository; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.janus.AtlasJanusGraph; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; @@ -53,20 +59,11 @@ import javax.inject.Inject; import java.io.IOException; import java.io.InputStream; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; +import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import static org.apache.atlas.repository.Constants.NAME; -import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; -import static org.apache.atlas.repository.Constants.SERVICE_ENTITY_TYPE; +import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_CATEGORY; import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_CONNECTION_QN; import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_IS_ENABLED; @@ -116,11 +113,15 @@ public class CachePolicyTransformerImpl { private PurposeCachePolicyTransformer purposeTransformer; private AtlasEntityHeader service; + private final ESBasedAuditRepository auditRepository; + + private final Map auditEventToDeltaChangeType; @Inject - public CachePolicyTransformerImpl(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { + public CachePolicyTransformerImpl(AtlasTypeRegistry typeRegistry, ESBasedAuditRepository auditRepository) throws AtlasBaseException { this.graph = new AtlasJanusGraph(); this.entityRetriever = new EntityGraphRetriever(graph, typeRegistry); + this.auditRepository = auditRepository; personaTransformer = new PersonaCachePolicyTransformer(entityRetriever); purposeTransformer = new PurposeCachePolicyTransformer(entityRetriever); @@ -131,13 +132,81 @@ public CachePolicyTransformerImpl(AtlasTypeRegistry typeRegistry) throws AtlasBa LOG.error("Failed to initialize discoveryService"); throw new AtlasBaseException(e.getCause()); } + + this.auditEventToDeltaChangeType = new HashMap<>(); + this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_CREATE, RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE); + this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_UPDATE, RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE); + this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_DELETE, RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE); } public AtlasEntityHeader getService() { return service; } - public ServicePolicies getPolicies(String serviceName, String pluginId, Long lastUpdatedTime) { + public ServicePolicies getPoliciesDelta(String serviceName, String pluginId, Long lastUpdatedTime) { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getPoliciesDelta." + serviceName); + + ServicePolicies servicePolicies = new ServicePolicies(); + + try { + servicePolicies.setServiceName(serviceName); + + service = getServiceEntity(serviceName); + servicePolicies.setPolicyVersion(-1L); + servicePolicies.setPolicyUpdateTime(new Date()); + + if (service != null) { + servicePolicies.setServiceName(serviceName); + servicePolicies.setServiceId(service.getGuid()); + + String serviceDefName = String.format(RESOURCE_SERVICE_DEF_PATTERN, serviceName); + servicePolicies.setServiceDef(getResourceAsObject(serviceDefName, RangerServiceDef.class)); + + List policiesDelta = getServicePoliciesDelta(service, 250, lastUpdatedTime); + servicePolicies.setPolicyDeltas(policiesDelta); + + + //Process tag based policies + String tagServiceName = (String) service.getAttribute(ATTR_SERVICE_TAG_SERVICE); + if (StringUtils.isNotEmpty(tagServiceName)) { + AtlasEntityHeader tagService = getServiceEntity(tagServiceName); + + if (tagService != null) { + List tagRangerPolicies = getServicePolicies(tagService, 0, null); + + TagPolicies tagPolicies = new TagPolicies(); + + tagPolicies.setServiceName(tagServiceName); + tagPolicies.setPolicyUpdateTime(new Date()); + tagPolicies.setServiceId(tagService.getGuid()); + tagPolicies.setPolicyVersion(-1L); + + String tagServiceDefName = String.format(RESOURCE_SERVICE_DEF_PATTERN, tagService.getAttribute(NAME)); + tagPolicies.setServiceDef(getResourceAsObject(tagServiceDefName, RangerServiceDef.class)); + + servicePolicies.setTagPolicies(tagPolicies); + servicePolicies.getTagPolicies().setPolicies(tagRangerPolicies); + LOG.info("PolicyDelta: {}: Found tag policies - {}", serviceName, tagRangerPolicies.size()); + } + } + + + + LOG.info("PolicyDelta: {}: Found {} policies", serviceName, policiesDelta.size()); + LOG.info("PolicyDelta: Found and set {} policies as delta and {} tag policies", servicePolicies.getPolicyDeltas().size(), servicePolicies.getTagPolicies().getPolicies().size()); + } + + } catch (Exception e) { + LOG.error("PolicyDelta: {}: ERROR in getPoliciesDelta {}: {}", serviceName, e.getMessage(), e); + return null; + } + + RequestContext.get().endMetricRecord(recorder); + return servicePolicies; + } + + + public ServicePolicies getPoliciesAll(String serviceName, String pluginId, Long lastUpdatedTime, Date latestEditTime) { //TODO: return only if updated AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getPolicies." + serviceName); @@ -151,7 +220,7 @@ public ServicePolicies getPolicies(String serviceName, String pluginId, Long las servicePolicies.setPolicyUpdateTime(new Date()); if (service != null) { - List allPolicies = getServicePolicies(service, 250); + List allPolicies = getServicePolicies(service, 250, latestEditTime); servicePolicies.setServiceName(serviceName); servicePolicies.setServiceId(service.getGuid()); @@ -165,7 +234,7 @@ public ServicePolicies getPolicies(String serviceName, String pluginId, Long las AtlasEntityHeader tagService = getServiceEntity(tagServiceName); if (tagService != null) { - allPolicies.addAll(getServicePolicies(tagService, 0)); + allPolicies.addAll(getServicePolicies(tagService, 0, latestEditTime)); TagPolicies tagPolicies = new TagPolicies(); @@ -195,7 +264,7 @@ public ServicePolicies getPolicies(String serviceName, String pluginId, Long las } } catch (Exception e) { - LOG.error("ERROR in getPolicies {}: ", e); + LOG.error("ERROR in getPolicies: ", e); return null; } @@ -203,14 +272,30 @@ public ServicePolicies getPolicies(String serviceName, String pluginId, Long las return servicePolicies; } - private List getServicePolicies(AtlasEntityHeader service, int batchSize) throws AtlasBaseException, IOException { + private List getServicePolicies(AtlasEntityHeader service, int batchSize, Date latestEditTime) throws AtlasBaseException, IOException, InterruptedException { List servicePolicies = new ArrayList<>(); + List atlasPolicies = new ArrayList<>(); String serviceName = (String) service.getAttribute("name"); String serviceType = (String) service.getAttribute("authServiceType"); - List atlasPolicies = getAtlasPolicies(serviceName, batchSize); + int maxAttempts = 5; + int sleepFor = 500; + for (int attempt = 0; attempt <= maxAttempts; attempt++) { + try { + atlasPolicies = getAtlasPolicies(serviceName, batchSize, latestEditTime, new ArrayList<>()); + break; + } catch (AtlasBaseException e) { + LOG.error("ES_SYNC_FIX: {}: ERROR in getServicePolicies: {}", serviceName, e.getMessage()); + TimeUnit.MILLISECONDS.sleep(sleepFor); + if (attempt == maxAttempts) { + throw e; + } + sleepFor *= 2; + } + } + LOG.info("ES_SYNC_FIX: {}: Moving to transform policies, size: {}", serviceName, atlasPolicies.size()); if (CollectionUtils.isNotEmpty(atlasPolicies)) { //transform policies servicePolicies = transformAtlasPoliciesToRangerPolicies(atlasPolicies, serviceType, serviceName); @@ -218,6 +303,83 @@ private List getServicePolicies(AtlasEntityHeader service, int bat return servicePolicies; } + private List getServicePoliciesDelta(AtlasEntityHeader service, int batchSize, Long lastUpdatedTime) throws AtlasBaseException, IOException { + + String serviceName = (String) service.getAttribute("name"); + String serviceType = (String) service.getAttribute("authServiceType"); + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getServicePoliciesWithDelta." + serviceName); + + List policyDeltas = new ArrayList<>(); + + // TODO: when getServicePolicies (without delta) is removed, merge the pagination for audit logs and policy fetch into one + List auditEvents = queryPoliciesAuditLogs(serviceName, lastUpdatedTime, batchSize); + Map policiesWithChangeType = new HashMap<>(); + for (EntityAuditEventV2 event : auditEvents) { + if (POLICY_ENTITY_TYPE.equals(event.getTypeName()) && !policiesWithChangeType.containsKey(event.getEntityId())) { + policiesWithChangeType.put(event.getEntityId(), event.getAction()); + } + } + LOG.info("PolicyDelta: {}: Total audit logs found = {}, events for {} ({}) = {}", serviceName, auditEvents.size(), POLICY_ENTITY_TYPE, policiesWithChangeType.size(), policiesWithChangeType); + if (policiesWithChangeType.isEmpty()) { + return policyDeltas; + } + + ArrayList policyGuids = new ArrayList<>(policiesWithChangeType.keySet()); + List atlasPolicies = getAtlasPolicies(serviceName, batchSize, null, policyGuids); + + List rangerPolicies = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(atlasPolicies)) { + rangerPolicies = transformAtlasPoliciesToRangerPolicies(atlasPolicies, serviceType, serviceName); + } + + for (RangerPolicy policy : rangerPolicies) { + Integer changeType = auditEventToDeltaChangeType.get(policiesWithChangeType.get(policy.getGuid())); + RangerPolicyDelta delta = new RangerPolicyDelta(policy.getId(), changeType, policy.getVersion(), policy); + policyDeltas.add(delta); + } + LOG.info("PolicyDelta: {}: atlas policies found = {}, delta created = {}", serviceName, atlasPolicies.size(), policyDeltas.size()); + RequestContext.get().endMetricRecord(recorder); + + return policyDeltas; + } + + private List queryPoliciesAuditLogs(String serviceName, Long afterTime, int batchSize) { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.queryPoliciesAuditLogs." + serviceName); + + List entityUpdateToWatch = new ArrayList<>(); + entityUpdateToWatch.add(POLICY_ENTITY_TYPE); + entityUpdateToWatch.add(PURPOSE_ENTITY_TYPE); + + AuditSearchParams parameters = new AuditSearchParams(); + Map dsl = getMap("size", batchSize); + + List> mustClauseList = new ArrayList<>(); + mustClauseList.add(getMap("terms", getMap("typeName", entityUpdateToWatch))); + afterTime = afterTime == -1 ? 0 : afterTime; + mustClauseList.add(getMap("range", getMap("created", getMap("gte", afterTime)))); + + List> sortList = new ArrayList<>(0); + sortList.add(getMap("created", getMap("order", "desc"))); + dsl.put("sort", sortList); + + dsl.put("query", getMap("bool", getMap("must", mustClauseList))); + + parameters.setDsl(dsl); + + List events = new ArrayList<>(); + try { + EntityAuditSearchResult result = auditRepository.searchEvents(parameters.getQueryString()); + if (result != null && !CollectionUtils.isEmpty(result.getEntityAudits())) { + events = result.getEntityAudits(); + } + } catch (AtlasBaseException e) { + LOG.error("ERROR in queryPoliciesAuditLogs while fetching entity audits {}: ", e.getMessage(), e); + } finally { + RequestContext.get().endMetricRecord(recorder); + } + return events; + } + private List transformAtlasPoliciesToRangerPolicies(List atlasPolicies, String serviceType, String serviceName) throws IOException, AtlasBaseException { @@ -417,13 +579,13 @@ private List getPolicyConditions(AtlasEntityHeader at return null; } - List conditions = (List) atlasPolicy.getAttribute("policyConditions"); + List> conditions = (List>) atlasPolicy.getAttribute("policyConditions"); - for (AtlasStruct condition : conditions) { + for (HashMap condition : conditions) { RangerPolicyItemCondition rangerCondition = new RangerPolicyItemCondition(); - rangerCondition.setType((String) condition.getAttribute("policyConditionType")); - rangerCondition.setValues((List) condition.getAttribute("policyConditionValues")); + rangerCondition.setType((String) condition.get("policyConditionType")); + rangerCondition.setValues((List) condition.get("policyConditionValues")); ret.add(rangerCondition); } @@ -452,7 +614,7 @@ private List getPolicyValiditySchedule(AtlasEntityHeader return ret; } - private List getAtlasPolicies(String serviceName, int batchSize) throws AtlasBaseException { + private List getAtlasPolicies(String serviceName, int batchSize, Date latestEditTime, List policyGuids) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl."+service+".getAtlasPolicies"); List ret = new ArrayList<>(); @@ -481,8 +643,13 @@ private List getAtlasPolicies(String serviceName, int batchSi List> mustClauseList = new ArrayList<>(); mustClauseList.add(getMap("term", getMap(ATTR_POLICY_SERVICE_NAME, serviceName))); + mustClauseList.add(getMap("term", getMap(ATTR_POLICY_IS_ENABLED, true))); mustClauseList.add(getMap("match", getMap("__state", Id.EntityState.ACTIVE))); + if (!policyGuids.isEmpty()) { + mustClauseList.add(getMap("terms", getMap("__guid", policyGuids))); + } + dsl.put("query", getMap("bool", getMap("must", mustClauseList))); List sortList = new ArrayList<>(0); @@ -509,13 +676,33 @@ private List getAtlasPolicies(String serviceName, int batchSi List headers = discoveryService.directIndexSearch(indexSearchParams).getEntities(); if (headers != null) { ret.addAll(headers); + LOG.info("ES_SYNC_FIX: {}: ======= Found result with {} policies", serviceName, headers.size()); } else { found = false; + LOG.info("ES_SYNC_FIX: {}: ======= Found result with null policies", serviceName); } from += size; } while (found && ret.size() % size == 0); + if (Objects.equals(serviceName, "atlas")) { + boolean latestEditFound = false; + Date latestEditTimeAvailable = null; + for (AtlasEntityHeader entity : ret) { + // LOG.info("ES_SYNC_FIX: {}: Looping on returned policies: {}, size: {}", serviceName, entity.getDisplayText(), ret.size()); + if (latestEditTime == null || entity.getUpdateTime().compareTo(latestEditTime) >= 0) { + LOG.info("ES_SYNC_FIX: {}: Found latest policy: {}, latestEditTime: {}, found policy time: {}", serviceName, entity.getDisplayText(), latestEditTime, entity.getUpdateTime()); + latestEditFound = true; + break; + } + latestEditTimeAvailable = entity.getUpdateTime(); + // LOG.info("ES_SYNC_FIX: {}: Checked for latest edit, entity: {}, latestEditTimeAvailable: {}", serviceName, entity.getDisplayText(), latestEditTimeAvailable); + } + if (latestEditTime != null && !latestEditFound) { + LOG.info("ES_SYNC_FIX: {}: Latest edit not found yet, policies: {}, latestEditTime: {}, latestEditTimeAvailable: {}", serviceName, ret.size(), latestEditTime, latestEditTimeAvailable); + throw new AtlasBaseException("Latest edit not found yet"); + } + } } finally { RequestContext.get().endMetricRecord(recorder); diff --git a/intg/src/main/java/org/apache/atlas/ApplicationProperties.java b/intg/src/main/java/org/apache/atlas/ApplicationProperties.java index 78b487a8dd..dc198ba2fa 100644 --- a/intg/src/main/java/org/apache/atlas/ApplicationProperties.java +++ b/intg/src/main/java/org/apache/atlas/ApplicationProperties.java @@ -70,6 +70,7 @@ public final class ApplicationProperties extends PropertiesConfiguration { public static final boolean DEFAULT_INDEX_RECOVERY = true; public static final AtlasRunMode DEFAULT_ATLAS_RUN_MODE = AtlasRunMode.PROD; public static final String INDEX_SEARCH_MAX_RESULT_SET_SIZE = "atlas.graph.index.search.max-result-set-size"; + public static final String DELTA_BASED_REFRESH = "atlas.authorizer.enable.delta_based_refresh"; public static final SimpleEntry DB_CACHE_CONF = new SimpleEntry<>("atlas.graph.cache.db-cache", "true"); public static final SimpleEntry DB_CACHE_CLEAN_WAIT_CONF = new SimpleEntry<>("atlas.graph.cache.db-cache-clean-wait", "20"); diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java index c5868cbfa3..a8a535cc7c 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java @@ -21,6 +21,7 @@ import org.apache.atlas.annotation.Timed; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.audit.AuditSearchParams; +import org.apache.atlas.model.audit.EntityAuditEventV2; import org.apache.atlas.model.audit.EntityAuditSearchResult; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.plugin.util.KeycloakUserStore; @@ -51,10 +52,7 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import static org.apache.atlas.policytransformer.CachePolicyTransformerImpl.ATTR_SERVICE_LAST_SYNC; import static org.apache.atlas.repository.Constants.PERSONA_ENTITY_TYPE; @@ -152,11 +150,12 @@ public ServicePolicies downloadPolicies(@PathParam("serviceName") final String s perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "AuthREST.downloadPolicies(serviceName="+serviceName+", pluginId="+pluginId+", lastUpdatedTime="+lastUpdatedTime+")"); } - if (!isPolicyUpdated(serviceName, lastUpdatedTime)) { + Long latestEditTime = getLastEditTime(serviceName, lastUpdatedTime); + if (latestEditTime == null) { return null; } - ServicePolicies ret = policyTransformer.getPolicies(serviceName, pluginId, lastUpdatedTime); + ServicePolicies ret = policyTransformer.getPoliciesAll(serviceName, pluginId, lastUpdatedTime, new Date(latestEditTime)); updateLastSync(serviceName); @@ -184,7 +183,7 @@ private void updateLastSync(String serviceName) { } } - private boolean isPolicyUpdated(String serviceName, long lastUpdatedTime) { + private Long getLastEditTime(String serviceName, long lastUpdatedTime) { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("AuthRest.isPolicyUpdated." + serviceName); List entityUpdateToWatch = new ArrayList<>(); @@ -199,26 +198,41 @@ private boolean isPolicyUpdated(String serviceName, long lastUpdatedTime) { mustClauseList.add(getMap("terms", getMap("typeName", entityUpdateToWatch))); lastUpdatedTime = lastUpdatedTime == -1 ? 0 : lastUpdatedTime; - mustClauseList.add(getMap("range", getMap("timestamp", getMap("gte", lastUpdatedTime)))); + mustClauseList.add(getMap("range", getMap("created", getMap("gte", lastUpdatedTime)))); dsl.put("query", getMap("bool", getMap("must", mustClauseList))); + List> sortList = new ArrayList<>(); + sortList.add(getMap("created", "desc")); + dsl.put("sort", sortList); + parameters.setDsl(dsl); + Long lastEditTime = 0L; // this timestamp is used to verify if the found policies are synced with any policy create or update op on cassandra try { EntityAuditSearchResult result = auditRepository.searchEvents(parameters.getQueryString()); - - if (result == null || CollectionUtils.isEmpty(result.getEntityAudits())) { - return false; + if (result != null) { + if (!CollectionUtils.isEmpty(result.getEntityAudits())) { + EntityAuditEventV2 lastAuditLog = result.getEntityAudits().get(0); + if (!EntityAuditEventV2.EntityAuditActionV2.getDeleteActions().contains(lastAuditLog.getAction()) && + lastAuditLog.getTypeName().equals(POLICY_ENTITY_TYPE) + ) { + lastEditTime = lastAuditLog.getTimestamp(); + } else { + LOG.info("ES_SYNC_FIX: {}: found delete action, so ignoring the last edit time: {}", serviceName, lastAuditLog.getTimestamp()); + } + } else { + lastEditTime = null; // no edits found + } } } catch (AtlasBaseException e) { LOG.error("ERROR in getPoliciesIfUpdated while fetching entity audits {}: ", e.getMessage()); - return true; } finally { RequestContext.get().endMetricRecord(recorder); + LOG.info("Last edit time for service {} is {}, dsl: {}", serviceName, lastEditTime, dsl); } - return true; + return lastEditTime; } private Map getMap(String key, Object value) { From b97cd8e461a46029daf9072bd62dd76566354f23 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Mon, 18 Nov 2024 17:02:35 +0530 Subject: [PATCH 02/31] Remove time based policy fetch retry during ES sync --- .../atlas/plugin/util/PolicyRefresher.java | 6 +-- .../CachePolicyTransformerImpl.java | 54 +++---------------- .../org/apache/atlas/web/rest/AuthREST.java | 40 +++++--------- 3 files changed, 23 insertions(+), 77 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java index b18906dde4..617dc10e7f 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -340,12 +340,10 @@ private ServicePolicies loadPolicyfromPolicyAdmin() throws RangerServiceNotFound CachePolicyTransformerImpl transformer = new CachePolicyTransformerImpl(plugIn.getTypeRegistry(), auditRepository); if (lastUpdatedTiemInMillis == -1) { svcPolicies = transformer.getPoliciesAll(serviceName, - restUtils.getPluginId(serviceName, plugIn.getAppId()), - lastUpdatedTiemInMillis, null); + restUtils.getPluginId(serviceName, plugIn.getAppId()), lastUpdatedTiemInMillis); } else if (this.enableDeltaBasedRefresh) { svcPolicies = transformer.getPoliciesDelta(serviceName, - restUtils.getPluginId(serviceName, plugIn.getAppId()), - lastUpdatedTiemInMillis); + restUtils.getPluginId(serviceName, plugIn.getAppId()), lastUpdatedTiemInMillis); } else { svcPolicies = atlasAuthAdminClient.getServicePoliciesIfUpdated(lastUpdatedTiemInMillis); } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index f092a488a6..4b04b6856d 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -18,7 +18,6 @@ package org.apache.atlas.policytransformer; -import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.AtlasException; import org.apache.atlas.RequestContext; import org.apache.atlas.discovery.EntityDiscoveryService; @@ -60,7 +59,6 @@ import java.io.IOException; import java.io.InputStream; import java.util.*; -import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; import static org.apache.atlas.repository.Constants.*; @@ -172,7 +170,7 @@ public ServicePolicies getPoliciesDelta(String serviceName, String pluginId, Lon AtlasEntityHeader tagService = getServiceEntity(tagServiceName); if (tagService != null) { - List tagRangerPolicies = getServicePolicies(tagService, 0, null); + List tagRangerPolicies = getServicePolicies(tagService, 0); TagPolicies tagPolicies = new TagPolicies(); @@ -206,7 +204,7 @@ public ServicePolicies getPoliciesDelta(String serviceName, String pluginId, Lon } - public ServicePolicies getPoliciesAll(String serviceName, String pluginId, Long lastUpdatedTime, Date latestEditTime) { + public ServicePolicies getPoliciesAll(String serviceName, String pluginId, Long lastUpdatedTime) { //TODO: return only if updated AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getPolicies." + serviceName); @@ -220,7 +218,7 @@ public ServicePolicies getPoliciesAll(String serviceName, String pluginId, Long servicePolicies.setPolicyUpdateTime(new Date()); if (service != null) { - List allPolicies = getServicePolicies(service, 250, latestEditTime); + List allPolicies = getServicePolicies(service, 250); servicePolicies.setServiceName(serviceName); servicePolicies.setServiceId(service.getGuid()); @@ -234,7 +232,7 @@ public ServicePolicies getPoliciesAll(String serviceName, String pluginId, Long AtlasEntityHeader tagService = getServiceEntity(tagServiceName); if (tagService != null) { - allPolicies.addAll(getServicePolicies(tagService, 0, latestEditTime)); + allPolicies.addAll(getServicePolicies(tagService, 0)); TagPolicies tagPolicies = new TagPolicies(); @@ -272,30 +270,14 @@ public ServicePolicies getPoliciesAll(String serviceName, String pluginId, Long return servicePolicies; } - private List getServicePolicies(AtlasEntityHeader service, int batchSize, Date latestEditTime) throws AtlasBaseException, IOException, InterruptedException { + private List getServicePolicies(AtlasEntityHeader service, int batchSize) throws AtlasBaseException, IOException { List servicePolicies = new ArrayList<>(); - List atlasPolicies = new ArrayList<>(); String serviceName = (String) service.getAttribute("name"); String serviceType = (String) service.getAttribute("authServiceType"); + List atlasPolicies = getAtlasPolicies(serviceName, batchSize, new ArrayList<>()); - int maxAttempts = 5; - int sleepFor = 500; - for (int attempt = 0; attempt <= maxAttempts; attempt++) { - try { - atlasPolicies = getAtlasPolicies(serviceName, batchSize, latestEditTime, new ArrayList<>()); - break; - } catch (AtlasBaseException e) { - LOG.error("ES_SYNC_FIX: {}: ERROR in getServicePolicies: {}", serviceName, e.getMessage()); - TimeUnit.MILLISECONDS.sleep(sleepFor); - if (attempt == maxAttempts) { - throw e; - } - sleepFor *= 2; - } - } - LOG.info("ES_SYNC_FIX: {}: Moving to transform policies, size: {}", serviceName, atlasPolicies.size()); if (CollectionUtils.isNotEmpty(atlasPolicies)) { //transform policies servicePolicies = transformAtlasPoliciesToRangerPolicies(atlasPolicies, serviceType, serviceName); @@ -325,7 +307,7 @@ private List getServicePoliciesDelta(AtlasEntityHeader servic } ArrayList policyGuids = new ArrayList<>(policiesWithChangeType.keySet()); - List atlasPolicies = getAtlasPolicies(serviceName, batchSize, null, policyGuids); + List atlasPolicies = getAtlasPolicies(serviceName, batchSize, policyGuids); List rangerPolicies = new ArrayList<>(); if (CollectionUtils.isNotEmpty(atlasPolicies)) { @@ -614,7 +596,7 @@ private List getPolicyValiditySchedule(AtlasEntityHeader return ret; } - private List getAtlasPolicies(String serviceName, int batchSize, Date latestEditTime, List policyGuids) throws AtlasBaseException { + private List getAtlasPolicies(String serviceName, int batchSize, List policyGuids) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl."+service+".getAtlasPolicies"); List ret = new ArrayList<>(); @@ -676,33 +658,13 @@ private List getAtlasPolicies(String serviceName, int batchSi List headers = discoveryService.directIndexSearch(indexSearchParams).getEntities(); if (headers != null) { ret.addAll(headers); - LOG.info("ES_SYNC_FIX: {}: ======= Found result with {} policies", serviceName, headers.size()); } else { found = false; - LOG.info("ES_SYNC_FIX: {}: ======= Found result with null policies", serviceName); } from += size; } while (found && ret.size() % size == 0); - if (Objects.equals(serviceName, "atlas")) { - boolean latestEditFound = false; - Date latestEditTimeAvailable = null; - for (AtlasEntityHeader entity : ret) { - // LOG.info("ES_SYNC_FIX: {}: Looping on returned policies: {}, size: {}", serviceName, entity.getDisplayText(), ret.size()); - if (latestEditTime == null || entity.getUpdateTime().compareTo(latestEditTime) >= 0) { - LOG.info("ES_SYNC_FIX: {}: Found latest policy: {}, latestEditTime: {}, found policy time: {}", serviceName, entity.getDisplayText(), latestEditTime, entity.getUpdateTime()); - latestEditFound = true; - break; - } - latestEditTimeAvailable = entity.getUpdateTime(); - // LOG.info("ES_SYNC_FIX: {}: Checked for latest edit, entity: {}, latestEditTimeAvailable: {}", serviceName, entity.getDisplayText(), latestEditTimeAvailable); - } - if (latestEditTime != null && !latestEditFound) { - LOG.info("ES_SYNC_FIX: {}: Latest edit not found yet, policies: {}, latestEditTime: {}, latestEditTimeAvailable: {}", serviceName, ret.size(), latestEditTime, latestEditTimeAvailable); - throw new AtlasBaseException("Latest edit not found yet"); - } - } } finally { RequestContext.get().endMetricRecord(recorder); diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java index a8a535cc7c..bb945ffa97 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java @@ -21,7 +21,6 @@ import org.apache.atlas.annotation.Timed; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.audit.AuditSearchParams; -import org.apache.atlas.model.audit.EntityAuditEventV2; import org.apache.atlas.model.audit.EntityAuditSearchResult; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.plugin.util.KeycloakUserStore; @@ -52,7 +51,10 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import static org.apache.atlas.policytransformer.CachePolicyTransformerImpl.ATTR_SERVICE_LAST_SYNC; import static org.apache.atlas.repository.Constants.PERSONA_ENTITY_TYPE; @@ -150,12 +152,11 @@ public ServicePolicies downloadPolicies(@PathParam("serviceName") final String s perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "AuthREST.downloadPolicies(serviceName="+serviceName+", pluginId="+pluginId+", lastUpdatedTime="+lastUpdatedTime+")"); } - Long latestEditTime = getLastEditTime(serviceName, lastUpdatedTime); - if (latestEditTime == null) { + if (!isPolicyUpdated(serviceName, lastUpdatedTime)) { return null; } - ServicePolicies ret = policyTransformer.getPoliciesAll(serviceName, pluginId, lastUpdatedTime, new Date(latestEditTime)); + ServicePolicies ret = policyTransformer.getPoliciesAll(serviceName, pluginId, lastUpdatedTime); updateLastSync(serviceName); @@ -183,7 +184,7 @@ private void updateLastSync(String serviceName) { } } - private Long getLastEditTime(String serviceName, long lastUpdatedTime) { + private boolean isPolicyUpdated(String serviceName, long lastUpdatedTime) { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("AuthRest.isPolicyUpdated." + serviceName); List entityUpdateToWatch = new ArrayList<>(); @@ -198,41 +199,26 @@ private Long getLastEditTime(String serviceName, long lastUpdatedTime) { mustClauseList.add(getMap("terms", getMap("typeName", entityUpdateToWatch))); lastUpdatedTime = lastUpdatedTime == -1 ? 0 : lastUpdatedTime; - mustClauseList.add(getMap("range", getMap("created", getMap("gte", lastUpdatedTime)))); + mustClauseList.add(getMap("range", getMap("timestamp", getMap("gte", lastUpdatedTime)))); dsl.put("query", getMap("bool", getMap("must", mustClauseList))); - List> sortList = new ArrayList<>(); - sortList.add(getMap("created", "desc")); - dsl.put("sort", sortList); - parameters.setDsl(dsl); - Long lastEditTime = 0L; // this timestamp is used to verify if the found policies are synced with any policy create or update op on cassandra try { EntityAuditSearchResult result = auditRepository.searchEvents(parameters.getQueryString()); - if (result != null) { - if (!CollectionUtils.isEmpty(result.getEntityAudits())) { - EntityAuditEventV2 lastAuditLog = result.getEntityAudits().get(0); - if (!EntityAuditEventV2.EntityAuditActionV2.getDeleteActions().contains(lastAuditLog.getAction()) && - lastAuditLog.getTypeName().equals(POLICY_ENTITY_TYPE) - ) { - lastEditTime = lastAuditLog.getTimestamp(); - } else { - LOG.info("ES_SYNC_FIX: {}: found delete action, so ignoring the last edit time: {}", serviceName, lastAuditLog.getTimestamp()); - } - } else { - lastEditTime = null; // no edits found - } + + if (result == null || CollectionUtils.isEmpty(result.getEntityAudits())) { + return false; } } catch (AtlasBaseException e) { LOG.error("ERROR in getPoliciesIfUpdated while fetching entity audits {}: ", e.getMessage()); + return true; } finally { RequestContext.get().endMetricRecord(recorder); - LOG.info("Last edit time for service {} is {}, dsl: {}", serviceName, lastEditTime, dsl); } - return lastEditTime; + return true; } private Map getMap(String key, Object value) { From 8fba21861f838915e56bc03f940664f13b9e583a Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Mon, 18 Nov 2024 17:05:28 +0530 Subject: [PATCH 03/31] Add missing changes for policy conditons --- .../policytransformer/CachePolicyTransformerImpl.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 4b04b6856d..3a97d0144e 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -29,6 +29,7 @@ import org.apache.atlas.model.discovery.AtlasSearchResult; import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.model.instance.AtlasStruct; import org.apache.atlas.plugin.model.RangerPolicyDelta; import org.apache.atlas.plugin.util.ServicePolicies; import org.apache.atlas.plugin.model.RangerPolicy; @@ -561,13 +562,13 @@ private List getPolicyConditions(AtlasEntityHeader at return null; } - List> conditions = (List>) atlasPolicy.getAttribute("policyConditions"); + List conditions = (List) atlasPolicy.getAttribute("policyConditions"); - for (HashMap condition : conditions) { + for (AtlasStruct condition : conditions) { RangerPolicyItemCondition rangerCondition = new RangerPolicyItemCondition(); - rangerCondition.setType((String) condition.get("policyConditionType")); - rangerCondition.setValues((List) condition.get("policyConditionValues")); + rangerCondition.setType((String) condition.getAttribute("policyConditionType")); + rangerCondition.setValues((List) condition.getAttribute("policyConditionValues")); ret.add(rangerCondition); } From a5394d29cf5ad90524a4af4a156cb4928a31fc77 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Mon, 25 Nov 2024 15:57:55 +0530 Subject: [PATCH 04/31] Cleanup ranger id of transformed policy to get atlas policy guid --- .../java/org/apache/atlas/plugin/model/RangerPolicy.java | 7 +++++++ .../apache/atlas/plugin/util/RangerPolicyDeltaUtil.java | 4 ++-- .../policytransformer/CachePolicyTransformerImpl.java | 5 +++-- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java index f81f8e2854..e1a6bda610 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java @@ -529,6 +529,13 @@ public void setIsDenyAllElse(Boolean isDenyAllElse) { this.isDenyAllElse = isDenyAllElse == null ? Boolean.FALSE : isDenyAllElse; } + public String getAtlasGuid() { + if (getGuid().length() > 36) { + return getGuid().substring(0, 36); + } + return getGuid(); + } + @Override public String toString( ) { StringBuilder sb = new StringBuilder(); diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java index b92ae004d4..e63345c10a 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java @@ -222,11 +222,11 @@ public static Boolean hasPolicyDeltas(final ServicePolicies servicePolicies) { if (isPoliciesExist && isPolicyDeltasExist) { LOG.warn("ServicePolicies contain both policies and policy-deltas!! Cannot build policy-engine from these servicePolicies. Please check server-side code!"); - LOG.warn("Downloaded ServicePolicies are [" + servicePolicies + "]"); + LOG.warn("Downloaded ServicePolicies for [" + servicePolicies.getServiceName() + "]"); ret = null; } else if (!isPoliciesExist && !isPolicyDeltasExist) { LOG.warn("ServicePolicies do not contain any policies or policy-deltas!! There are no material changes in the policies."); - LOG.warn("Downloaded ServicePolicies are [" + servicePolicies + "]"); + LOG.warn("Downloaded ServicePolicies for [" + servicePolicies.getServiceName() + "]"); ret = null; } else { ret = isPolicyDeltasExist; diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 3a97d0144e..ebe8812db1 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -182,9 +182,9 @@ public ServicePolicies getPoliciesDelta(String serviceName, String pluginId, Lon String tagServiceDefName = String.format(RESOURCE_SERVICE_DEF_PATTERN, tagService.getAttribute(NAME)); tagPolicies.setServiceDef(getResourceAsObject(tagServiceDefName, RangerServiceDef.class)); + tagPolicies.setPolicies(tagRangerPolicies); servicePolicies.setTagPolicies(tagPolicies); - servicePolicies.getTagPolicies().setPolicies(tagRangerPolicies); LOG.info("PolicyDelta: {}: Found tag policies - {}", serviceName, tagRangerPolicies.size()); } } @@ -308,6 +308,7 @@ private List getServicePoliciesDelta(AtlasEntityHeader servic } ArrayList policyGuids = new ArrayList<>(policiesWithChangeType.keySet()); + // this will have less policies as deleted won't be found List atlasPolicies = getAtlasPolicies(serviceName, batchSize, policyGuids); List rangerPolicies = new ArrayList<>(); @@ -316,7 +317,7 @@ private List getServicePoliciesDelta(AtlasEntityHeader servic } for (RangerPolicy policy : rangerPolicies) { - Integer changeType = auditEventToDeltaChangeType.get(policiesWithChangeType.get(policy.getGuid())); + Integer changeType = auditEventToDeltaChangeType.get(policiesWithChangeType.get(policy.getAtlasGuid())); RangerPolicyDelta delta = new RangerPolicyDelta(policy.getId(), changeType, policy.getVersion(), policy); policyDeltas.add(delta); } From d803b86bfdc81bb4ef3c0ffec004ad3f0f82d57a Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Mon, 25 Nov 2024 16:50:02 +0530 Subject: [PATCH 05/31] Use App Configuration instead of reading the config again --- .../java/org/apache/atlas/plugin/util/PolicyRefresher.java | 4 ++-- intg/src/main/java/org/apache/atlas/AtlasConfiguration.java | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java index 617dc10e7f..4a5dc552bd 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -22,6 +22,7 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasException; import org.apache.atlas.authz.admin.client.AtlasAuthAdminClient; import org.apache.atlas.policytransformer.CachePolicyTransformerImpl; @@ -114,10 +115,9 @@ public PolicyRefresher(RangerBasePlugin plugIn) { this.pollingIntervalMs = pluginConfig.getLong(propertyPrefix + ".policy.pollIntervalMs", 30 * 1000); try { - this.atlasConfig = ApplicationProperties.get(); + this.enableDeltaBasedRefresh = AtlasConfiguration.DELTA_BASED_REFRESH_ENABLED.getBoolean(); this.auditRepository = new ESBasedAuditRepository(atlasConfig); this.auditRepository.start(); - this.enableDeltaBasedRefresh = this.atlasConfig.getBoolean(DELTA_BASED_REFRESH, false); } catch (AtlasException e) { LOG.error("PolicyDelta: Error while reading atlas configuration", e); this.enableDeltaBasedRefresh = false; diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index df2bca7860..f8c8e82063 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -120,6 +120,8 @@ public enum AtlasConfiguration { ATLAS_MAINTENANCE_MODE("atlas.maintenance.mode", false), + DELTA_BASED_REFRESH_ENABLED("atlas.authorizer.enable.delta_based_refresh", false), + ATLAS_UD_RELATIONSHIPS_MAX_COUNT("atlas.ud.relationship.max.count", 100); From 04304e03e2435ce9fe4dd756bba06ee52e3e49a2 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Tue, 26 Nov 2024 13:13:30 +0530 Subject: [PATCH 06/31] Enable build on branch to test before merging --- .github/workflows/maven.yml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index f8a09b5589..38fc6c2898 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,6 +26,7 @@ on: - development - master - lineageondemand + - ns/fix/delta-refresh jobs: build: @@ -48,7 +49,9 @@ jobs: restore-keys: ${{ runner.os }}-m2 - name: Get branch name - run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" + run: | + echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" + echo "##[set-output name=branch_lower;]$(echo ${GITHUB_REF#refs/heads/} | awk '{gsub("/", "-"); print tolower($0)}')" id: get_branch - name: Create Maven Settings @@ -102,8 +105,8 @@ jobs: provenance: true push: true tags: | - ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch }}:latest - ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch }}:${{ steps.get_version.outputs.version }} + ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch_lower }}:latest + ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch_lower }}:${{ steps.get_version.outputs.version }} - name: Scan Image uses: aquasecurity/trivy-action@master From 70bed029914e00268214591e9c3a7b429f325645 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Thu, 28 Nov 2024 00:20:55 +0530 Subject: [PATCH 07/31] Move audit log checks to authrest itself --- .../admin/client/AtlasAuthAdminClient.java | 2 +- .../admin/client/AtlasAuthRESTClient.java | 12 ++-- .../atlas/plugin/util/PolicyRefresher.java | 20 ++---- .../CachePolicyTransformerImpl.java | 49 ++++++++------- .../org/apache/atlas/web/rest/AuthREST.java | 61 ++++++++++++++++--- 5 files changed, 92 insertions(+), 52 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/authz/admin/client/AtlasAuthAdminClient.java b/auth-agents-common/src/main/java/org/apache/atlas/authz/admin/client/AtlasAuthAdminClient.java index a79407ba9c..7d2f675d9e 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/authz/admin/client/AtlasAuthAdminClient.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/authz/admin/client/AtlasAuthAdminClient.java @@ -8,7 +8,7 @@ public interface AtlasAuthAdminClient { void init(RangerPluginConfig config); - ServicePolicies getServicePoliciesIfUpdated(long lastUpdatedTimeInMillis) throws Exception; + ServicePolicies getServicePoliciesIfUpdated(long lastUpdatedTimeInMillis, boolean usePolicyDelta) throws Exception; RangerRoles getRolesIfUpdated(long lastUpdatedTimeInMillis) throws Exception; diff --git a/auth-agents-common/src/main/java/org/apache/atlas/authz/admin/client/AtlasAuthRESTClient.java b/auth-agents-common/src/main/java/org/apache/atlas/authz/admin/client/AtlasAuthRESTClient.java index 10c97008c3..c2a7dafd83 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/authz/admin/client/AtlasAuthRESTClient.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/authz/admin/client/AtlasAuthRESTClient.java @@ -32,6 +32,7 @@ public class AtlasAuthRESTClient implements AtlasAuthAdminClient { private static final String PARAM_LAST_UPDATED_TIME = "lastUpdatedTime"; private static final String PARAM_PLUGIN_ID = "pluginId"; + private static final String PARAM_USE_POLICY_DELTA = "usePolicyDelta"; @Override public void init(RangerPluginConfig config) { @@ -66,20 +67,20 @@ public String getPluginId(String serviceName, String appId) { } @Override - public ServicePolicies getServicePoliciesIfUpdated(long lastUpdatedTimeInMillis) throws Exception { - URI uri = buildURI("/download/policies/" + serviceName, lastUpdatedTimeInMillis); + public ServicePolicies getServicePoliciesIfUpdated(long lastUpdatedTimeInMillis, boolean usePolicyDelta) throws Exception { + URI uri = buildURI("/download/policies/" + serviceName, lastUpdatedTimeInMillis, usePolicyDelta); return sendRequestAndGetResponse(uri, ServicePolicies.class); } @Override public RangerRoles getRolesIfUpdated(long lastUpdatedTimeInMillis) throws Exception { - URI uri = buildURI("/download/roles/" + serviceName, lastUpdatedTimeInMillis); + URI uri = buildURI("/download/roles/" + serviceName, lastUpdatedTimeInMillis, false); return sendRequestAndGetResponse(uri, RangerRoles.class); } @Override public RangerUserStore getUserStoreIfUpdated(long lastUpdatedTimeInMillis) throws Exception { - URI uri = buildURI("/download/users/" + serviceName, lastUpdatedTimeInMillis); + URI uri = buildURI("/download/users/" + serviceName, lastUpdatedTimeInMillis, false); return sendRequestAndGetResponse(uri, RangerUserStore.class); } @@ -114,13 +115,14 @@ private T sendRequestAndGetResponse(URI uri, Class responseClass) throws return null; } - private URI buildURI(String path, long lastUpdatedTimeInMillis) throws URISyntaxException { + private URI buildURI(String path, long lastUpdatedTimeInMillis, boolean usePolicyDelta) throws URISyntaxException { return new URIBuilder() .setScheme(SCHEME) .setHost(adminUrl) .setPath(path) .setParameter(PARAM_LAST_UPDATED_TIME, String.valueOf(lastUpdatedTimeInMillis)) .setParameter(PARAM_PLUGIN_ID, pluginId) + .setParameter(PARAM_USE_POLICY_DELTA, String.valueOf(usePolicyDelta)) .build(); } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java index 4a5dc552bd..5946dc7d79 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -21,7 +21,6 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; -import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasException; import org.apache.atlas.authz.admin.client.AtlasAuthAdminClient; @@ -45,8 +44,6 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; -import static org.apache.atlas.ApplicationProperties.DELTA_BASED_REFRESH; - public class PolicyRefresher extends Thread { private static final Log LOG = LogFactory.getLog(PolicyRefresher.class); @@ -335,20 +332,15 @@ private ServicePolicies loadPolicyfromPolicyAdmin() throws RangerServiceNotFound try { - if (serviceName.equals("atlas") && plugIn.getTypeRegistry() != null) { + if (serviceName.equals("atlas") && plugIn.getTypeRegistry() != null && lastUpdatedTiemInMillis == -1) { RangerRESTUtils restUtils = new RangerRESTUtils(); CachePolicyTransformerImpl transformer = new CachePolicyTransformerImpl(plugIn.getTypeRegistry(), auditRepository); - if (lastUpdatedTiemInMillis == -1) { - svcPolicies = transformer.getPoliciesAll(serviceName, - restUtils.getPluginId(serviceName, plugIn.getAppId()), lastUpdatedTiemInMillis); - } else if (this.enableDeltaBasedRefresh) { - svcPolicies = transformer.getPoliciesDelta(serviceName, - restUtils.getPluginId(serviceName, plugIn.getAppId()), lastUpdatedTiemInMillis); - } else { - svcPolicies = atlasAuthAdminClient.getServicePoliciesIfUpdated(lastUpdatedTiemInMillis); - } + + svcPolicies = transformer.getPoliciesAll(serviceName, + restUtils.getPluginId(serviceName, plugIn.getAppId()), + lastUpdatedTiemInMillis); } else { - svcPolicies = atlasAuthAdminClient.getServicePoliciesIfUpdated(lastUpdatedTiemInMillis); + svcPolicies = atlasAuthAdminClient.getServicePoliciesIfUpdated(lastUpdatedTiemInMillis, this.enableDeltaBasedRefresh); } boolean isUpdated = svcPolicies != null; diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index ebe8812db1..6715103ce0 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -142,7 +142,7 @@ public AtlasEntityHeader getService() { return service; } - public ServicePolicies getPoliciesDelta(String serviceName, String pluginId, Long lastUpdatedTime) { + public ServicePolicies createPoliciesWithDelta(String serviceName, Map policyChanges, List atlasPolicies) { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getPoliciesDelta." + serviceName); ServicePolicies servicePolicies = new ServicePolicies(); @@ -161,7 +161,7 @@ public ServicePolicies getPoliciesDelta(String serviceName, String pluginId, Lon String serviceDefName = String.format(RESOURCE_SERVICE_DEF_PATTERN, serviceName); servicePolicies.setServiceDef(getResourceAsObject(serviceDefName, RangerServiceDef.class)); - List policiesDelta = getServicePoliciesDelta(service, 250, lastUpdatedTime); + List policiesDelta = getRangerPolicyDelta(service, policyChanges, atlasPolicies); servicePolicies.setPolicyDeltas(policiesDelta); @@ -189,8 +189,6 @@ public ServicePolicies getPoliciesDelta(String serviceName, String pluginId, Lon } } - - LOG.info("PolicyDelta: {}: Found {} policies", serviceName, policiesDelta.size()); LOG.info("PolicyDelta: Found and set {} policies as delta and {} tag policies", servicePolicies.getPolicyDeltas().size(), servicePolicies.getTagPolicies().getPolicies().size()); } @@ -286,38 +284,23 @@ private List getServicePolicies(AtlasEntityHeader service, int bat return servicePolicies; } - private List getServicePoliciesDelta(AtlasEntityHeader service, int batchSize, Long lastUpdatedTime) throws AtlasBaseException, IOException { - + private List getRangerPolicyDelta(AtlasEntityHeader service, Map policyChanges, List atlasPolicies) throws AtlasBaseException, IOException { String serviceName = (String) service.getAttribute("name"); String serviceType = (String) service.getAttribute("authServiceType"); - AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getServicePoliciesWithDelta." + serviceName); + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getRangerPolicyDelta." + serviceName); List policyDeltas = new ArrayList<>(); - - // TODO: when getServicePolicies (without delta) is removed, merge the pagination for audit logs and policy fetch into one - List auditEvents = queryPoliciesAuditLogs(serviceName, lastUpdatedTime, batchSize); - Map policiesWithChangeType = new HashMap<>(); - for (EntityAuditEventV2 event : auditEvents) { - if (POLICY_ENTITY_TYPE.equals(event.getTypeName()) && !policiesWithChangeType.containsKey(event.getEntityId())) { - policiesWithChangeType.put(event.getEntityId(), event.getAction()); - } - } - LOG.info("PolicyDelta: {}: Total audit logs found = {}, events for {} ({}) = {}", serviceName, auditEvents.size(), POLICY_ENTITY_TYPE, policiesWithChangeType.size(), policiesWithChangeType); - if (policiesWithChangeType.isEmpty()) { + if (policyChanges.isEmpty()) { return policyDeltas; } - ArrayList policyGuids = new ArrayList<>(policiesWithChangeType.keySet()); - // this will have less policies as deleted won't be found - List atlasPolicies = getAtlasPolicies(serviceName, batchSize, policyGuids); - List rangerPolicies = new ArrayList<>(); if (CollectionUtils.isNotEmpty(atlasPolicies)) { rangerPolicies = transformAtlasPoliciesToRangerPolicies(atlasPolicies, serviceType, serviceName); } for (RangerPolicy policy : rangerPolicies) { - Integer changeType = auditEventToDeltaChangeType.get(policiesWithChangeType.get(policy.getAtlasGuid())); + Integer changeType = auditEventToDeltaChangeType.get(policyChanges.get(policy.getAtlasGuid())); RangerPolicyDelta delta = new RangerPolicyDelta(policy.getId(), changeType, policy.getVersion(), policy); policyDeltas.add(delta); } @@ -364,6 +347,26 @@ private List queryPoliciesAuditLogs(String serviceName, Long return events; } + public ServicePolicies extractAndTransformPolicyDelta(String serviceName, List events) { + Map policyChanges = new HashMap<>(); + for (EntityAuditEventV2 event : events) { + if (POLICY_ENTITY_TYPE.equals(event.getTypeName()) && !policyChanges.containsKey(event.getEntityId())) { + policyChanges.put(event.getEntityId(), event.getAction()); + } + } + + List atlasPolicies = new ArrayList<>(); + for (EntityAuditEventV2 event : events) { + AtlasEntityHeader policy = event.getEntityHeader(); + if (policy != null) { + atlasPolicies.add(policy); + } + } + LOG.info("PolicyDelta: {}: Found {} policy changes with {} policies", serviceName, policyChanges.size(), atlasPolicies.size()); + + return createPoliciesWithDelta(serviceName, policyChanges, atlasPolicies); + } + private List transformAtlasPoliciesToRangerPolicies(List atlasPolicies, String serviceType, String serviceName) throws IOException, AtlasBaseException { diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java index bb945ffa97..14ed3b1101 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java @@ -21,6 +21,7 @@ import org.apache.atlas.annotation.Timed; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.audit.AuditSearchParams; +import org.apache.atlas.model.audit.EntityAuditEventV2; import org.apache.atlas.model.audit.EntityAuditSearchResult; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.plugin.util.KeycloakUserStore; @@ -51,10 +52,7 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import static org.apache.atlas.policytransformer.CachePolicyTransformerImpl.ATTR_SERVICE_LAST_SYNC; import static org.apache.atlas.repository.Constants.PERSONA_ENTITY_TYPE; @@ -144,6 +142,7 @@ public RangerUserStore downloadUserStore(@PathParam("serviceName") final String @Timed public ServicePolicies downloadPolicies(@PathParam("serviceName") final String serviceName, @QueryParam("pluginId") String pluginId, + @DefaultValue("false") @QueryParam("usePolicyDelta") boolean usePolicyDelta, @DefaultValue("0") @QueryParam("lastUpdatedTime") Long lastUpdatedTime) throws AtlasBaseException { AtlasPerfTracer perf = null; @@ -152,12 +151,16 @@ public ServicePolicies downloadPolicies(@PathParam("serviceName") final String s perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "AuthREST.downloadPolicies(serviceName="+serviceName+", pluginId="+pluginId+", lastUpdatedTime="+lastUpdatedTime+")"); } - if (!isPolicyUpdated(serviceName, lastUpdatedTime)) { - return null; + ServicePolicies ret; + if (usePolicyDelta) { + List auditEvents = getPolicyAuditLogs(serviceName, lastUpdatedTime); + ret = policyTransformer.extractAndTransformPolicyDelta(serviceName, auditEvents); + } else { + if (!isPolicyUpdated(serviceName, lastUpdatedTime)) { + return null; + } + ret = policyTransformer.getPoliciesAll(serviceName, pluginId, lastUpdatedTime); } - - ServicePolicies ret = policyTransformer.getPoliciesAll(serviceName, pluginId, lastUpdatedTime); - updateLastSync(serviceName); return ret; @@ -166,6 +169,46 @@ public ServicePolicies downloadPolicies(@PathParam("serviceName") final String s } } + private List getPolicyAuditLogs(String serviceName, long lastUpdatedTime) { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("AuthREST.getPolicyAuditLogs." + serviceName); + + List entityUpdateToWatch = new ArrayList<>(); + entityUpdateToWatch.add(POLICY_ENTITY_TYPE); + entityUpdateToWatch.add(PERSONA_ENTITY_TYPE); + entityUpdateToWatch.add(PURPOSE_ENTITY_TYPE); + + AuditSearchParams parameters = new AuditSearchParams(); + Map dsl = getMap("size", 100); + + List> mustClauseList = new ArrayList<>(); + mustClauseList.add(getMap("terms", getMap("typeName", entityUpdateToWatch))); + + lastUpdatedTime = lastUpdatedTime == -1 ? 0 : lastUpdatedTime; + mustClauseList.add(getMap("range", getMap("timestamp", getMap("gte", lastUpdatedTime)))); + + dsl.put("query", getMap("bool", getMap("must", mustClauseList))); + + parameters.setDsl(dsl); + parameters.setAttributes(new HashSet<>(Arrays.asList( + "qualifiedName", + "name" + ))); + + List events = new ArrayList<>(); + try { + EntityAuditSearchResult result = auditRepository.searchEvents(parameters.getQueryString()); + if (result != null && !CollectionUtils.isEmpty(result.getEntityAudits())) { + events = result.getEntityAudits(); + } + } catch (AtlasBaseException e) { + LOG.error("ERROR in getPolicyAuditLogs while fetching entity audits {}: ", e.getMessage()); + } finally { + RequestContext.get().endMetricRecord(recorder); + } + + return events; + } + private void updateLastSync(String serviceName) { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("AuthRest.updateLastSync." + serviceName); From 49e0deb477d94ec8600b3072bc4eff0af52e3c82 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Thu, 28 Nov 2024 12:01:55 +0530 Subject: [PATCH 08/31] Remove auditRepo from policy refresher and transformer --- .../atlas/plugin/util/PolicyRefresher.java | 14 ++----- .../CachePolicyTransformerImpl.java | 41 +------------------ 2 files changed, 4 insertions(+), 51 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java index 5946dc7d79..b9331a547c 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -68,9 +68,7 @@ public class PolicyRefresher extends Thread { private long lastActivationTimeInMillis; private boolean policiesSetInPlugin; private boolean serviceDefSetInPlugin; - private Configuration atlasConfig; private boolean enableDeltaBasedRefresh; - private ESBasedAuditRepository auditRepository; public PolicyRefresher(RangerBasePlugin plugIn) { @@ -111,14 +109,8 @@ public PolicyRefresher(RangerBasePlugin plugIn) { this.userStoreProvider = new RangerUserStoreProvider(getServiceType(), appId, getServiceName(), atlasAuthAdminClient, cacheDir, pluginConfig); this.pollingIntervalMs = pluginConfig.getLong(propertyPrefix + ".policy.pollIntervalMs", 30 * 1000); - try { - this.enableDeltaBasedRefresh = AtlasConfiguration.DELTA_BASED_REFRESH_ENABLED.getBoolean(); - this.auditRepository = new ESBasedAuditRepository(atlasConfig); - this.auditRepository.start(); - } catch (AtlasException e) { - LOG.error("PolicyDelta: Error while reading atlas configuration", e); - this.enableDeltaBasedRefresh = false; - } + this.enableDeltaBasedRefresh = AtlasConfiguration.DELTA_BASED_REFRESH_ENABLED.getBoolean(); + LOG.info("PolicyRefresher(serviceName=" + serviceName + ") - delta based policy refresh is enabled"); setName("PolicyRefresher(serviceName=" + serviceName + ")-" + getId()); @@ -334,7 +326,7 @@ private ServicePolicies loadPolicyfromPolicyAdmin() throws RangerServiceNotFound if (serviceName.equals("atlas") && plugIn.getTypeRegistry() != null && lastUpdatedTiemInMillis == -1) { RangerRESTUtils restUtils = new RangerRESTUtils(); - CachePolicyTransformerImpl transformer = new CachePolicyTransformerImpl(plugIn.getTypeRegistry(), auditRepository); + CachePolicyTransformerImpl transformer = new CachePolicyTransformerImpl(plugIn.getTypeRegistry()); svcPolicies = transformer.getPoliciesAll(serviceName, restUtils.getPluginId(serviceName, plugIn.getAppId()), diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 6715103ce0..3edd214e3b 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -112,15 +112,13 @@ public class CachePolicyTransformerImpl { private PurposeCachePolicyTransformer purposeTransformer; private AtlasEntityHeader service; - private final ESBasedAuditRepository auditRepository; private final Map auditEventToDeltaChangeType; @Inject - public CachePolicyTransformerImpl(AtlasTypeRegistry typeRegistry, ESBasedAuditRepository auditRepository) throws AtlasBaseException { + public CachePolicyTransformerImpl(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { this.graph = new AtlasJanusGraph(); this.entityRetriever = new EntityGraphRetriever(graph, typeRegistry); - this.auditRepository = auditRepository; personaTransformer = new PersonaCachePolicyTransformer(entityRetriever); purposeTransformer = new PurposeCachePolicyTransformer(entityRetriever); @@ -310,43 +308,6 @@ private List getRangerPolicyDelta(AtlasEntityHeader service, return policyDeltas; } - private List queryPoliciesAuditLogs(String serviceName, Long afterTime, int batchSize) { - AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.queryPoliciesAuditLogs." + serviceName); - - List entityUpdateToWatch = new ArrayList<>(); - entityUpdateToWatch.add(POLICY_ENTITY_TYPE); - entityUpdateToWatch.add(PURPOSE_ENTITY_TYPE); - - AuditSearchParams parameters = new AuditSearchParams(); - Map dsl = getMap("size", batchSize); - - List> mustClauseList = new ArrayList<>(); - mustClauseList.add(getMap("terms", getMap("typeName", entityUpdateToWatch))); - afterTime = afterTime == -1 ? 0 : afterTime; - mustClauseList.add(getMap("range", getMap("created", getMap("gte", afterTime)))); - - List> sortList = new ArrayList<>(0); - sortList.add(getMap("created", getMap("order", "desc"))); - dsl.put("sort", sortList); - - dsl.put("query", getMap("bool", getMap("must", mustClauseList))); - - parameters.setDsl(dsl); - - List events = new ArrayList<>(); - try { - EntityAuditSearchResult result = auditRepository.searchEvents(parameters.getQueryString()); - if (result != null && !CollectionUtils.isEmpty(result.getEntityAudits())) { - events = result.getEntityAudits(); - } - } catch (AtlasBaseException e) { - LOG.error("ERROR in queryPoliciesAuditLogs while fetching entity audits {}: ", e.getMessage(), e); - } finally { - RequestContext.get().endMetricRecord(recorder); - } - return events; - } - public ServicePolicies extractAndTransformPolicyDelta(String serviceName, List events) { Map policyChanges = new HashMap<>(); for (EntityAuditEventV2 event : events) { From 2aad6a5c429fb6352d7decd9ca3da20db4d4f7b8 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Fri, 29 Nov 2024 15:11:06 +0530 Subject: [PATCH 09/31] Revert to ealier way to fetching changed policies Policy object in audit log was querying all policies one by one --- .../atlas/plugin/model/RangerPolicyDelta.java | 4 +-- .../plugin/service/RangerBasePlugin.java | 2 +- .../atlas/plugin/util/PolicyRefresher.java | 3 ++ .../CachePolicyTransformerImpl.java | 34 ++++++++----------- .../org/apache/atlas/web/rest/AuthREST.java | 13 +++---- 5 files changed, 24 insertions(+), 32 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java index def6c6c0dd..b5f9b58233 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java @@ -19,8 +19,8 @@ package org.apache.atlas.plugin.model; -import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonIgnore; -import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonIgnore; +import com.fasterxml.jackson.annotation.JsonInclude; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java index fb44584e44..76ee3c3e1f 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java @@ -314,7 +314,7 @@ public void setPolicies(ServicePolicies policies) { Boolean hasPolicyDeltas = RangerPolicyDeltaUtil.hasPolicyDeltas(policies); if (hasPolicyDeltas == null) { - LOG.info("Downloaded policies do not require policy change !! [" + policies + "]"); + LOG.info("Downloaded policies do not require policy change !! [" + (policies.getPolicies() != null ? policies.getPolicies().size() : 0) + "]"); if (this.policyEngine == null) { diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java index b9331a547c..5b806fd08e 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -324,7 +324,9 @@ private ServicePolicies loadPolicyfromPolicyAdmin() throws RangerServiceNotFound try { + if (serviceName.equals("atlas") && plugIn.getTypeRegistry() != null && lastUpdatedTiemInMillis == -1) { + LOG.info("PolicyRefresher(serviceName=" + serviceName + "): loading all policies for first time"); RangerRESTUtils restUtils = new RangerRESTUtils(); CachePolicyTransformerImpl transformer = new CachePolicyTransformerImpl(plugIn.getTypeRegistry()); @@ -332,6 +334,7 @@ private ServicePolicies loadPolicyfromPolicyAdmin() throws RangerServiceNotFound restUtils.getPluginId(serviceName, plugIn.getAppId()), lastUpdatedTiemInMillis); } else { + LOG.info("PolicyRefresher(serviceName=" + serviceName + "): loading delta policies from last known version=" + lastKnownVersion + ", lastUpdatedTime=" + lastUpdatedTiemInMillis); svcPolicies = atlasAuthAdminClient.getServicePoliciesIfUpdated(lastUpdatedTiemInMillis, this.enableDeltaBasedRefresh); } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 3edd214e3b..c0a8894cd5 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -22,10 +22,8 @@ import org.apache.atlas.RequestContext; import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.audit.AuditSearchParams; import org.apache.atlas.model.audit.EntityAuditEventV2; import org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2; -import org.apache.atlas.model.audit.EntityAuditSearchResult; import org.apache.atlas.model.discovery.AtlasSearchResult; import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.instance.AtlasEntityHeader; @@ -42,7 +40,6 @@ import org.apache.atlas.plugin.model.RangerServiceDef; import org.apache.atlas.plugin.model.RangerValiditySchedule; import org.apache.atlas.plugin.util.ServicePolicies.TagPolicies; -import org.apache.atlas.repository.audit.ESBasedAuditRepository; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.janus.AtlasJanusGraph; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; @@ -97,12 +94,13 @@ public class CachePolicyTransformerImpl { private static final String ATTR_POLICY_GROUPS = "policyGroups"; private static final String ATTR_POLICY_USERS = "policyUsers"; private static final String ATTR_POLICY_ROLES = "policyRoles"; - private static final String ATTR_POLICY_VALIDITY = "policyValiditySchedule"; - private static final String ATTR_POLICY_CONDITIONS = "policyConditions"; - private static final String ATTR_POLICY_MASK_TYPE = "policyMaskType"; + public static final String ATTR_POLICY_VALIDITY = "policyValiditySchedule"; + public static final String ATTR_POLICY_CONDITIONS = "policyConditions"; + public static final String ATTR_POLICY_MASK_TYPE = "policyMaskType"; private static final String RESOURCE_SERVICE_DEF_PATH = "/service-defs/"; private static final String RESOURCE_SERVICE_DEF_PATTERN = RESOURCE_SERVICE_DEF_PATH + "atlas-servicedef-%s.json"; + public static final int POLICY_BATCH_SIZE = 250; private EntityDiscoveryService discoveryService; private AtlasGraph graph; @@ -140,7 +138,7 @@ public AtlasEntityHeader getService() { return service; } - public ServicePolicies createPoliciesWithDelta(String serviceName, Map policyChanges, List atlasPolicies) { + public ServicePolicies getPoliciesDelta(String serviceName, Map policyChanges) { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getPoliciesDelta." + serviceName); ServicePolicies servicePolicies = new ServicePolicies(); @@ -159,7 +157,7 @@ public ServicePolicies createPoliciesWithDelta(String serviceName, Map policiesDelta = getRangerPolicyDelta(service, policyChanges, atlasPolicies); + List policiesDelta = getRangerPolicyDelta(service, policyChanges); servicePolicies.setPolicyDeltas(policiesDelta); @@ -215,7 +213,7 @@ public ServicePolicies getPoliciesAll(String serviceName, String pluginId, Long servicePolicies.setPolicyUpdateTime(new Date()); if (service != null) { - List allPolicies = getServicePolicies(service, 250); + List allPolicies = getServicePolicies(service, POLICY_BATCH_SIZE); servicePolicies.setServiceName(serviceName); servicePolicies.setServiceId(service.getGuid()); @@ -282,7 +280,7 @@ private List getServicePolicies(AtlasEntityHeader service, int bat return servicePolicies; } - private List getRangerPolicyDelta(AtlasEntityHeader service, Map policyChanges, List atlasPolicies) throws AtlasBaseException, IOException { + private List getRangerPolicyDelta(AtlasEntityHeader service, Map policyChanges) throws AtlasBaseException, IOException { String serviceName = (String) service.getAttribute("name"); String serviceType = (String) service.getAttribute("authServiceType"); AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getRangerPolicyDelta." + serviceName); @@ -292,6 +290,9 @@ private List getRangerPolicyDelta(AtlasEntityHeader service, return policyDeltas; } + ArrayList policyGuids = new ArrayList<>(policyChanges.keySet()); + List atlasPolicies = getAtlasPolicies(serviceName, POLICY_BATCH_SIZE, policyGuids); + List rangerPolicies = new ArrayList<>(); if (CollectionUtils.isNotEmpty(atlasPolicies)) { rangerPolicies = transformAtlasPoliciesToRangerPolicies(atlasPolicies, serviceType, serviceName); @@ -308,7 +309,7 @@ private List getRangerPolicyDelta(AtlasEntityHeader service, return policyDeltas; } - public ServicePolicies extractAndTransformPolicyDelta(String serviceName, List events) { + public Map createPolicyChangeMap(String serviceName, List events) { Map policyChanges = new HashMap<>(); for (EntityAuditEventV2 event : events) { if (POLICY_ENTITY_TYPE.equals(event.getTypeName()) && !policyChanges.containsKey(event.getEntityId())) { @@ -316,16 +317,9 @@ public ServicePolicies extractAndTransformPolicyDelta(String serviceName, List atlasPolicies = new ArrayList<>(); - for (EntityAuditEventV2 event : events) { - AtlasEntityHeader policy = event.getEntityHeader(); - if (policy != null) { - atlasPolicies.add(policy); - } - } - LOG.info("PolicyDelta: {}: Found {} policy changes with {} policies", serviceName, policyChanges.size(), atlasPolicies.size()); + LOG.info("PolicyDelta: {}: Found {} policy changes in {} policies", serviceName, events.size(), policyChanges.size()); - return createPoliciesWithDelta(serviceName, policyChanges, atlasPolicies); + return policyChanges; } private List transformAtlasPoliciesToRangerPolicies(List atlasPolicies, diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java index 14ed3b1101..e4ac2552bd 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java @@ -154,7 +154,8 @@ public ServicePolicies downloadPolicies(@PathParam("serviceName") final String s ServicePolicies ret; if (usePolicyDelta) { List auditEvents = getPolicyAuditLogs(serviceName, lastUpdatedTime); - ret = policyTransformer.extractAndTransformPolicyDelta(serviceName, auditEvents); + Map policyChanges = policyTransformer.createPolicyChangeMap(serviceName, auditEvents); + ret = policyTransformer.getPoliciesDelta(serviceName, policyChanges); } else { if (!isPolicyUpdated(serviceName, lastUpdatedTime)) { return null; @@ -174,8 +175,6 @@ private List getPolicyAuditLogs(String serviceName, long las List entityUpdateToWatch = new ArrayList<>(); entityUpdateToWatch.add(POLICY_ENTITY_TYPE); - entityUpdateToWatch.add(PERSONA_ENTITY_TYPE); - entityUpdateToWatch.add(PURPOSE_ENTITY_TYPE); AuditSearchParams parameters = new AuditSearchParams(); Map dsl = getMap("size", 100); @@ -187,16 +186,12 @@ private List getPolicyAuditLogs(String serviceName, long las mustClauseList.add(getMap("range", getMap("timestamp", getMap("gte", lastUpdatedTime)))); dsl.put("query", getMap("bool", getMap("must", mustClauseList))); - parameters.setDsl(dsl); - parameters.setAttributes(new HashSet<>(Arrays.asList( - "qualifiedName", - "name" - ))); List events = new ArrayList<>(); try { - EntityAuditSearchResult result = auditRepository.searchEvents(parameters.getQueryString()); + String query = parameters.getQueryString(); + EntityAuditSearchResult result = auditRepository.searchEvents(query); // attributes are not getting passed in query if (result != null && !CollectionUtils.isEmpty(result.getEntityAudits())) { events = result.getEntityAudits(); } From dbf2854f6fbdc3e9689e22ae2cdddf1a0388ed8e Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Mon, 2 Dec 2024 14:51:33 +0530 Subject: [PATCH 10/31] Use last policy updated at time as last update time --- .../atlas/plugin/model/RangerPolicy.java | 4 +- .../atlas/plugin/util/PolicyRefresher.java | 7 +-- .../atlas/plugin/util/ServicePolicies.java | 44 ++++++++++++++++--- .../CachePolicyTransformerImpl.java | 11 +++-- .../org/apache/atlas/web/rest/AuthREST.java | 3 +- 5 files changed, 51 insertions(+), 18 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java index e1a6bda610..3a4e52fd3b 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java @@ -19,8 +19,9 @@ package org.apache.atlas.plugin.model; +import com.fasterxml.jackson.annotation.JsonIgnore; import org.apache.commons.collections.CollectionUtils; -import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude; import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; @@ -529,6 +530,7 @@ public void setIsDenyAllElse(Boolean isDenyAllElse) { this.isDenyAllElse = isDenyAllElse == null ? Boolean.FALSE : isDenyAllElse; } + @JsonIgnore public String getAtlasGuid() { if (getGuid().length() > 36) { return getGuid().substring(0, 36); diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java index 5b806fd08e..b74741cfad 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -22,11 +22,8 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; import org.apache.atlas.AtlasConfiguration; -import org.apache.atlas.AtlasException; import org.apache.atlas.authz.admin.client.AtlasAuthAdminClient; import org.apache.atlas.policytransformer.CachePolicyTransformerImpl; -import org.apache.atlas.repository.audit.ESBasedAuditRepository; -import org.apache.commons.configuration.Configuration; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -281,7 +278,7 @@ private void loadPolicy() { serviceDefSetInPlugin = false; setLastActivationTimeInMillis(System.currentTimeMillis()); lastKnownVersion = svcPolicies.getPolicyVersion() != null ? svcPolicies.getPolicyVersion() : -1L; - lastUpdatedTiemInMillis = svcPolicies.getPolicyUpdateTime() != null ? svcPolicies.getPolicyUpdateTime().getTime() : -1L; + lastUpdatedTiemInMillis = svcPolicies.getLatestUpdateTime() != null ? svcPolicies.getLatestUpdateTime().getTime() : -1L; } else { if (!policiesSetInPlugin && !serviceDefSetInPlugin) { plugIn.setPolicies(null); @@ -402,7 +399,7 @@ private ServicePolicies loadFromCache() { } lastKnownVersion = policies.getPolicyVersion() == null ? -1 : policies.getPolicyVersion().longValue(); - lastUpdatedTiemInMillis = policies.getPolicyUpdateTime() == null ? -1 : policies.getPolicyUpdateTime().getTime(); + lastUpdatedTiemInMillis = policies.getLatestUpdateTime() == null ? -1 : policies.getLatestUpdateTime().getTime(); } } catch (Exception excp) { LOG.error("failed to load policies from cache file " + cacheFile.getAbsolutePath(), excp); diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java index 1a4dd81f29..2fb01d9e5f 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java @@ -20,10 +20,11 @@ package org.apache.atlas.plugin.util; +import com.fasterxml.jackson.annotation.JsonIgnore; import org.apache.commons.collections.MapUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonInclude; import org.apache.atlas.plugin.model.RangerPolicy; import org.apache.atlas.plugin.model.RangerPolicyDelta; import org.apache.atlas.plugin.model.RangerServiceDef; @@ -33,12 +34,7 @@ import javax.xml.bind.annotation.XmlAccessType; import javax.xml.bind.annotation.XmlAccessorType; import javax.xml.bind.annotation.XmlRootElement; -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; @JsonInclude(JsonInclude.Include.NON_NULL) @XmlRootElement @@ -180,6 +176,40 @@ public String toString() { + "securityZones=" + securityZones ; } + + @JsonIgnore + public Date getLatestUpdateTime() { + Date lastestUpdateTime = new Date(0L); + + if (policies != null && !policies.isEmpty()) { + for (RangerPolicy policy : policies) { + if (policy.getUpdateTime() != null && policy.getUpdateTime().after(lastestUpdateTime)) { + lastestUpdateTime = policy.getUpdateTime(); + } + } + } + + if (tagPolicies != null && tagPolicies.getPolicies() != null) { + for (RangerPolicy policy : tagPolicies.getPolicies()) { + if (policy.getUpdateTime() != null && policy.getUpdateTime().after(lastestUpdateTime)) { + lastestUpdateTime = policy.getUpdateTime(); + } + } + } + + if (policyDeltas != null && !policyDeltas.isEmpty()) { + for (RangerPolicyDelta delta : policyDeltas) { + if (delta.getPolicy() != null && delta.getPolicy().getUpdateTime() != null && delta.getPolicy().getUpdateTime().after(lastestUpdateTime)) { + lastestUpdateTime = delta.getPolicy().getUpdateTime(); + } + } + } + if (Objects.equals(lastestUpdateTime, new Date(0L))) { + lastestUpdateTime = null; + } + return lastestUpdateTime; + } + public List getPolicyDeltas() { return this.policyDeltas; } public void setPolicyDeltas(List policyDeltas) { this.policyDeltas = policyDeltas; } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index c0a8894cd5..8d4edad4e5 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -138,17 +138,18 @@ public AtlasEntityHeader getService() { return service; } - public ServicePolicies getPoliciesDelta(String serviceName, Map policyChanges) { + public ServicePolicies getPoliciesDelta(String serviceName, Map policyChanges, long lastAuditEventTime) { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getPoliciesDelta." + serviceName); ServicePolicies servicePolicies = new ServicePolicies(); try { - servicePolicies.setServiceName(serviceName); - service = getServiceEntity(serviceName); + servicePolicies.setServiceName(serviceName); servicePolicies.setPolicyVersion(-1L); - servicePolicies.setPolicyUpdateTime(new Date()); + + Date policyUpdateTime = lastAuditEventTime > 0 ? new Date(lastAuditEventTime) : new Date(); + servicePolicies.setPolicyUpdateTime(policyUpdateTime); if (service != null) { servicePolicies.setServiceName(serviceName); @@ -678,6 +679,8 @@ private RangerPolicy getRangerPolicy(AtlasEntityHeader atlasPolicy, String servi policy.setGuid(atlasPolicy.getGuid()); policy.setCreatedBy(atlasPolicy.getCreatedBy()); policy.setCreateTime(atlasPolicy.getCreateTime()); + policy.setUpdatedBy(atlasPolicy.getUpdatedBy()); + policy.setUpdateTime(atlasPolicy.getUpdateTime()); policy.setIsEnabled(getIsPolicyEnabled(atlasPolicy)); policy.setConditions(getPolicyConditions(atlasPolicy)); diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java index e4ac2552bd..3f552331a9 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java @@ -154,8 +154,9 @@ public ServicePolicies downloadPolicies(@PathParam("serviceName") final String s ServicePolicies ret; if (usePolicyDelta) { List auditEvents = getPolicyAuditLogs(serviceName, lastUpdatedTime); + long lastEventTime = auditEvents.stream().mapToLong(EntityAuditEventV2::getTimestamp).max().orElse(0); Map policyChanges = policyTransformer.createPolicyChangeMap(serviceName, auditEvents); - ret = policyTransformer.getPoliciesDelta(serviceName, policyChanges); + ret = policyTransformer.getPoliciesDelta(serviceName, policyChanges, lastEventTime); } else { if (!isPolicyUpdated(serviceName, lastUpdatedTime)) { return null; From 0e678ed855b3b50c7005267b9620df672628edf2 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Mon, 2 Dec 2024 15:50:07 +0530 Subject: [PATCH 11/31] Use current time as last update time for policy refresher In local it was creating problem as all policies are updated on startup and query fetches policies which has greater or equal to that timestamp which include all of them. --- .../java/org/apache/atlas/plugin/util/PolicyRefresher.java | 3 ++- .../atlas/policytransformer/CachePolicyTransformerImpl.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java index b74741cfad..b8d7daf4df 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -37,6 +37,7 @@ import java.io.FileWriter; import java.io.Reader; import java.io.Writer; +import java.util.Date; import java.util.Timer; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; @@ -278,7 +279,7 @@ private void loadPolicy() { serviceDefSetInPlugin = false; setLastActivationTimeInMillis(System.currentTimeMillis()); lastKnownVersion = svcPolicies.getPolicyVersion() != null ? svcPolicies.getPolicyVersion() : -1L; - lastUpdatedTiemInMillis = svcPolicies.getLatestUpdateTime() != null ? svcPolicies.getLatestUpdateTime().getTime() : -1L; + lastUpdatedTiemInMillis = svcPolicies.getPolicyUpdateTime() != null ? svcPolicies.getPolicyUpdateTime().getTime() : -1L; } else { if (!policiesSetInPlugin && !serviceDefSetInPlugin) { plugIn.setPolicies(null); diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 8d4edad4e5..f8eef3c8e0 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -148,7 +148,8 @@ public ServicePolicies getPoliciesDelta(String serviceName, Map 0 ? new Date(lastAuditEventTime) : new Date(); + // Date policyUpdateTime = lastAuditEventTime > 0 ? new Date(lastAuditEventTime) : new Date(); + Date policyUpdateTime = new Date(); servicePolicies.setPolicyUpdateTime(policyUpdateTime); if (service != null) { From d8e7ef60366a86144adb7a5cc1a1e770e52bdfc2 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Tue, 3 Dec 2024 17:11:55 +0530 Subject: [PATCH 12/31] Use pagination while fetching audit logs --- .../CachePolicyTransformerImpl.java | 12 +++++---- .../org/apache/atlas/web/rest/AuthREST.java | 26 ++++++++++++++----- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index f8eef3c8e0..7439fb9aec 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -169,7 +169,7 @@ public ServicePolicies getPoliciesDelta(String serviceName, Map tagRangerPolicies = getServicePolicies(tagService, 0); +// List tagRangerPolicies = getServicePolicies(tagService, 0); TagPolicies tagPolicies = new TagPolicies(); @@ -180,15 +180,14 @@ public ServicePolicies getPoliciesDelta(String serviceName, Map getAtlasPolicies(String serviceName, int batchSi Map dsl = getMap("size", 0); List> mustClauseList = new ArrayList<>(); - mustClauseList.add(getMap("term", getMap(ATTR_POLICY_SERVICE_NAME, serviceName))); + mustClauseList.add(getMap("term", getMap(ATTR_POLICY_IS_ENABLED, true))); mustClauseList.add(getMap("match", getMap("__state", Id.EntityState.ACTIVE))); if (!policyGuids.isEmpty()) { mustClauseList.add(getMap("terms", getMap("__guid", policyGuids))); + } else { + // no service filter required if guids are provided + mustClauseList.add(getMap("term", getMap(ATTR_POLICY_SERVICE_NAME, serviceName))); } dsl.put("query", getMap("bool", getMap("must", mustClauseList))); diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java index 3f552331a9..25e144344c 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java @@ -163,6 +163,10 @@ public ServicePolicies downloadPolicies(@PathParam("serviceName") final String s } ret = policyTransformer.getPoliciesAll(serviceName, pluginId, lastUpdatedTime); } + LOG.info("downloadPolicies: serviceName={}, lastUpdatedTime={}, policies fetched={} delta fetched={}", serviceName, + lastUpdatedTime > 0 ? new Date(lastUpdatedTime) : lastUpdatedTime, + ret != null && ret.getPolicies() != null ? ret.getPolicies().size() : 0, + ret != null && ret.getPolicyDeltas() != null ? ret.getPolicyDeltas().size() : 0); updateLastSync(serviceName); return ret; @@ -184,18 +188,26 @@ private List getPolicyAuditLogs(String serviceName, long las mustClauseList.add(getMap("terms", getMap("typeName", entityUpdateToWatch))); lastUpdatedTime = lastUpdatedTime == -1 ? 0 : lastUpdatedTime; - mustClauseList.add(getMap("range", getMap("timestamp", getMap("gte", lastUpdatedTime)))); + mustClauseList.add(getMap("range", getMap("created", getMap("gte", lastUpdatedTime)))); dsl.put("query", getMap("bool", getMap("must", mustClauseList))); - parameters.setDsl(dsl); + + int from = 0; + int size = 100; List events = new ArrayList<>(); try { - String query = parameters.getQueryString(); - EntityAuditSearchResult result = auditRepository.searchEvents(query); // attributes are not getting passed in query - if (result != null && !CollectionUtils.isEmpty(result.getEntityAudits())) { - events = result.getEntityAudits(); - } + do { + dsl.put("from", from); + dsl.put("size", size); + parameters.setDsl(dsl); + String query = parameters.getQueryString(); + EntityAuditSearchResult result = auditRepository.searchEvents(query); // attributes are not getting passed in query + if (result != null && !CollectionUtils.isEmpty(result.getEntityAudits())) { + events = result.getEntityAudits(); + } + from += size; + } while (events.size() == size); } catch (AtlasBaseException e) { LOG.error("ERROR in getPolicyAuditLogs while fetching entity audits {}: ", e.getMessage()); } finally { From 94e53f182e9bf29b93e181f4490ba8a3ca618277 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Tue, 3 Dec 2024 17:22:31 +0530 Subject: [PATCH 13/31] fix: emmit audits in case of policy update by accesscontrol resource update --- .../graph/v2/preprocessor/AuthPolicyPreProcessor.java | 7 +++++++ .../v2/preprocessor/accesscontrol/PersonaPreProcessor.java | 1 + .../v2/preprocessor/accesscontrol/PurposePreProcessor.java | 2 ++ 3 files changed, 10 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java index 58fb516564..c87d1bc6e1 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java @@ -139,6 +139,9 @@ private void processCreatePolicy(AtlasStruct entity) throws AtlasBaseException { policy.setAttribute(ATTR_POLICY_USERS, new ArrayList<>()); policy.setAttribute(ATTR_POLICY_GROUPS, new ArrayList<>()); + if(parentEntity != null) { + policy.setAttribute(ATTR_POLICY_IS_ENABLED, getIsAccessControlEnabled(parentEntity)); + } //create ES alias aliasStore.updateAlias(parent, policy); @@ -155,6 +158,10 @@ private void processCreatePolicy(AtlasStruct entity) throws AtlasBaseException { policy.setAttribute(ATTR_POLICY_RESOURCES, policyResources); + if(parentEntity != null) { + policy.setAttribute(ATTR_POLICY_IS_ENABLED, getIsAccessControlEnabled(parentEntity)); + } + //create ES alias aliasStore.updateAlias(parent, policy); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java index 8a2c8b1945..e6f815e36a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java @@ -205,6 +205,7 @@ private void updatePoliciesIsEnabledAttr(EntityMutationContext context, AtlasEnt policyToBeUpdated.setAttribute(ATTR_POLICY_IS_ENABLED, enable); context.addUpdated(policyToBeUpdated.getGuid(), policyToBeUpdated, entityType, policyVertex); + RequestContext.get().cacheDifferentialEntity(policyToBeUpdated); } } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PurposePreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PurposePreProcessor.java index ca8fcb803f..3dbf342feb 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PurposePreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PurposePreProcessor.java @@ -167,6 +167,7 @@ private void processUpdatePurpose(EntityMutationContext context, policyToBeUpdated.setAttribute(ATTR_POLICY_RESOURCES, newTagsResources); context.addUpdated(policyToBeUpdated.getGuid(), policyToBeUpdated, entityType, policyVertex); + RequestContext.get().cacheDifferentialEntity(policyToBeUpdated); existingPurposeExtInfo.addReferredEntity(policyToBeUpdated); } @@ -199,6 +200,7 @@ private void updatePoliciesIsEnabledAttr(EntityMutationContext context, AtlasEnt policyToBeUpdated.setAttribute(ATTR_POLICY_IS_ENABLED, enable); context.addUpdated(policyToBeUpdated.getGuid(), policyToBeUpdated, entityType, policyVertex); + RequestContext.get().cacheDifferentialEntity(policyToBeUpdated); } } } From cb35dd8181a9f4c030f05d04eeaf78e37b36eb4a Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Tue, 3 Dec 2024 17:59:38 +0530 Subject: [PATCH 14/31] Use policy udpate time as last refresh time --- .../policytransformer/CachePolicyTransformerImpl.java | 4 ++-- .../src/main/java/org/apache/atlas/web/rest/AuthREST.java | 7 ++++++- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 7439fb9aec..5623120dc0 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -148,8 +148,8 @@ public ServicePolicies getPoliciesDelta(String serviceName, Map 0 ? new Date(lastAuditEventTime) : new Date(); - Date policyUpdateTime = new Date(); + Date policyUpdateTime = lastAuditEventTime > 0 ? new Date(lastAuditEventTime) : new Date(); +// Date policyUpdateTime = new Date(); servicePolicies.setPolicyUpdateTime(policyUpdateTime); if (service != null) { diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java index 25e144344c..8d8ebaf036 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java @@ -154,7 +154,8 @@ public ServicePolicies downloadPolicies(@PathParam("serviceName") final String s ServicePolicies ret; if (usePolicyDelta) { List auditEvents = getPolicyAuditLogs(serviceName, lastUpdatedTime); - long lastEventTime = auditEvents.stream().mapToLong(EntityAuditEventV2::getTimestamp).max().orElse(0); + long lastEventTime = auditEvents.isEmpty() ? 0 : auditEvents.get(auditEvents.size() - 1).getCreated(); + long lastEventTime2 = auditEvents.stream().mapToLong(EntityAuditEventV2::getTimestamp).max().orElse(0); Map policyChanges = policyTransformer.createPolicyChangeMap(serviceName, auditEvents); ret = policyTransformer.getPoliciesDelta(serviceName, policyChanges, lastEventTime); } else { @@ -192,6 +193,10 @@ private List getPolicyAuditLogs(String serviceName, long las dsl.put("query", getMap("bool", getMap("must", mustClauseList))); + List> sortClause = new ArrayList<>(); + sortClause.add(getMap("created", getMap("order", "asc"))); + dsl.put("sort", sortClause); + int from = 0; int size = 100; From 32e075fa28c9621d1467f665892bfd699a29ee3e Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Wed, 4 Dec 2024 17:09:55 +0530 Subject: [PATCH 15/31] Handle delete policy event --- .../atlas/plugin/util/PolicyRefresher.java | 2 +- .../plugin/util/RangerPolicyDeltaUtil.java | 9 +++-- .../CachePolicyTransformerImpl.java | 40 ++++++++++++------- .../org/apache/atlas/web/rest/AuthREST.java | 7 +++- 4 files changed, 37 insertions(+), 21 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java index b8d7daf4df..5f9351be26 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -108,7 +108,7 @@ public PolicyRefresher(RangerBasePlugin plugIn) { this.pollingIntervalMs = pluginConfig.getLong(propertyPrefix + ".policy.pollIntervalMs", 30 * 1000); this.enableDeltaBasedRefresh = AtlasConfiguration.DELTA_BASED_REFRESH_ENABLED.getBoolean(); - LOG.info("PolicyRefresher(serviceName=" + serviceName + ") - delta based policy refresh is enabled"); + LOG.info("PolicyRefresher(serviceName=" + serviceName + ") - delta based policy refresh enabled="+this.enableDeltaBasedRefresh); setName("PolicyRefresher(serviceName=" + serviceName + ")-" + getId()); diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java index e63345c10a..0c8267b5c7 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java @@ -91,7 +91,10 @@ public static List applyDeltas(List policies, List applyDeltas(List policies, List 1) { - LOG.warn("Unexpected: found no policy or multiple policies for CHANGE_TYPE_POLICY_DELETE: " + Arrays.toString(deletedPolicies.toArray())); + if (CollectionUtils.isEmpty(deletedPolicies)) { + LOG.warn("Unexpected: found no policy for CHANGE_TYPE_POLICY_DELETE: " + Arrays.toString(deletedPolicies.toArray())); } break; } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 5623120dc0..1edf210162 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -60,17 +60,7 @@ import java.util.stream.Collectors; import static org.apache.atlas.repository.Constants.*; -import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_CATEGORY; -import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_CONNECTION_QN; -import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_IS_ENABLED; -import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_PRIORITY; -import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_SERVICE_NAME; -import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_SUB_CATEGORY; -import static org.apache.atlas.repository.util.AccessControlUtils.POLICY_CATEGORY_DATAMESH; -import static org.apache.atlas.repository.util.AccessControlUtils.POLICY_CATEGORY_PERSONA; -import static org.apache.atlas.repository.util.AccessControlUtils.POLICY_CATEGORY_PURPOSE; -import static org.apache.atlas.repository.util.AccessControlUtils.getIsPolicyEnabled; -import static org.apache.atlas.repository.util.AccessControlUtils.getPolicyCategory; +import static org.apache.atlas.repository.util.AccessControlUtils.*; @Component public class CachePolicyTransformerImpl { @@ -304,7 +294,29 @@ private List getRangerPolicyDelta(AtlasEntityHeader service, RangerPolicyDelta delta = new RangerPolicyDelta(policy.getId(), changeType, policy.getVersion(), policy); policyDeltas.add(delta); } - LOG.info("PolicyDelta: {}: atlas policies found = {}, delta created = {}", serviceName, atlasPolicies.size(), policyDeltas.size()); + + // handle delete changes separately as they won't be present in atlas policies + List deletedPolicyDeltas = new ArrayList<>(); + for (String policyGuid : policyGuids) { + if (policyChanges.get(policyGuid) == EntityAuditActionV2.ENTITY_DELETE) { + RangerPolicy deletedPolicy = new RangerPolicy(); + deletedPolicy.setGuid(policyGuid); + deletedPolicy.setService(serviceName); + deletedPolicy.setServiceType(serviceType); + RangerPolicyDelta deletedPolicyDelta = new RangerPolicyDelta( + deletedPolicy.getId(), + auditEventToDeltaChangeType.get(EntityAuditActionV2.ENTITY_DELETE), + deletedPolicy.getVersion(), + deletedPolicy + ); + deletedPolicyDeltas.add(deletedPolicyDelta); + } + } + + policyDeltas.addAll(deletedPolicyDeltas); + + LOG.info("PolicyDelta: {}: atlas policies found={}, delta created={}, including deleted policies={}", + serviceName, atlasPolicies.size(), policyDeltas.size(), deletedPolicyDeltas.size()); RequestContext.get().endMetricRecord(recorder); return policyDeltas; @@ -585,15 +597,13 @@ private List getAtlasPolicies(String serviceName, int batchSi Map dsl = getMap("size", 0); List> mustClauseList = new ArrayList<>(); - - mustClauseList.add(getMap("term", getMap(ATTR_POLICY_IS_ENABLED, true))); mustClauseList.add(getMap("match", getMap("__state", Id.EntityState.ACTIVE))); if (!policyGuids.isEmpty()) { mustClauseList.add(getMap("terms", getMap("__guid", policyGuids))); } else { - // no service filter required if guids are provided mustClauseList.add(getMap("term", getMap(ATTR_POLICY_SERVICE_NAME, serviceName))); + mustClauseList.add(getMap("term", getMap(ATTR_POLICY_IS_ENABLED, true))); } dsl.put("query", getMap("bool", getMap("must", mustClauseList))); diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java index 8d8ebaf036..c668aa5c90 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java @@ -155,7 +155,10 @@ public ServicePolicies downloadPolicies(@PathParam("serviceName") final String s if (usePolicyDelta) { List auditEvents = getPolicyAuditLogs(serviceName, lastUpdatedTime); long lastEventTime = auditEvents.isEmpty() ? 0 : auditEvents.get(auditEvents.size() - 1).getCreated(); - long lastEventTime2 = auditEvents.stream().mapToLong(EntityAuditEventV2::getTimestamp).max().orElse(0); + LOG.info("PolicyDelta: serviceName={}, lastUpdatedTime={}, audit events found={}", serviceName, lastEventTime, auditEvents.size()); + if (auditEvents.isEmpty()) { + return null; + } Map policyChanges = policyTransformer.createPolicyChangeMap(serviceName, auditEvents); ret = policyTransformer.getPoliciesDelta(serviceName, policyChanges, lastEventTime); } else { @@ -189,7 +192,7 @@ private List getPolicyAuditLogs(String serviceName, long las mustClauseList.add(getMap("terms", getMap("typeName", entityUpdateToWatch))); lastUpdatedTime = lastUpdatedTime == -1 ? 0 : lastUpdatedTime; - mustClauseList.add(getMap("range", getMap("created", getMap("gte", lastUpdatedTime)))); + mustClauseList.add(getMap("range", getMap("created", getMap("gt", lastUpdatedTime)))); dsl.put("query", getMap("bool", getMap("must", mustClauseList))); From eca292554f79f0e05e854ca8162675ade211695b Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Thu, 5 Dec 2024 12:54:40 +0530 Subject: [PATCH 16/31] Use individual imports and remove dead code --- .github/workflows/maven.yml | 9 ++--- .../atlas/plugin/util/PolicyRefresher.java | 2 +- .../atlas/plugin/util/ServicePolicies.java | 35 +------------------ .../CachePolicyTransformerImpl.java | 33 ++++++++++++----- .../apache/atlas/ApplicationProperties.java | 1 - .../org/apache/atlas/web/rest/AuthREST.java | 6 +++- 6 files changed, 34 insertions(+), 52 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 38fc6c2898..f8a09b5589 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,7 +26,6 @@ on: - development - master - lineageondemand - - ns/fix/delta-refresh jobs: build: @@ -49,9 +48,7 @@ jobs: restore-keys: ${{ runner.os }}-m2 - name: Get branch name - run: | - echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" - echo "##[set-output name=branch_lower;]$(echo ${GITHUB_REF#refs/heads/} | awk '{gsub("/", "-"); print tolower($0)}')" + run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" id: get_branch - name: Create Maven Settings @@ -105,8 +102,8 @@ jobs: provenance: true push: true tags: | - ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch_lower }}:latest - ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch_lower }}:${{ steps.get_version.outputs.version }} + ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch }}:latest + ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch }}:${{ steps.get_version.outputs.version }} - name: Scan Image uses: aquasecurity/trivy-action@master diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java index 5f9351be26..ed5b1a3687 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -400,7 +400,7 @@ private ServicePolicies loadFromCache() { } lastKnownVersion = policies.getPolicyVersion() == null ? -1 : policies.getPolicyVersion().longValue(); - lastUpdatedTiemInMillis = policies.getLatestUpdateTime() == null ? -1 : policies.getLatestUpdateTime().getTime(); + lastUpdatedTiemInMillis = policies.getPolicyUpdateTime() == null ? -1 : policies.getPolicyUpdateTime().getTime(); } } catch (Exception excp) { LOG.error("failed to load policies from cache file " + cacheFile.getAbsolutePath(), excp); diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java index 2fb01d9e5f..77717e1e08 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java @@ -177,39 +177,6 @@ public String toString() { ; } - @JsonIgnore - public Date getLatestUpdateTime() { - Date lastestUpdateTime = new Date(0L); - - if (policies != null && !policies.isEmpty()) { - for (RangerPolicy policy : policies) { - if (policy.getUpdateTime() != null && policy.getUpdateTime().after(lastestUpdateTime)) { - lastestUpdateTime = policy.getUpdateTime(); - } - } - } - - if (tagPolicies != null && tagPolicies.getPolicies() != null) { - for (RangerPolicy policy : tagPolicies.getPolicies()) { - if (policy.getUpdateTime() != null && policy.getUpdateTime().after(lastestUpdateTime)) { - lastestUpdateTime = policy.getUpdateTime(); - } - } - } - - if (policyDeltas != null && !policyDeltas.isEmpty()) { - for (RangerPolicyDelta delta : policyDeltas) { - if (delta.getPolicy() != null && delta.getPolicy().getUpdateTime() != null && delta.getPolicy().getUpdateTime().after(lastestUpdateTime)) { - lastestUpdateTime = delta.getPolicy().getUpdateTime(); - } - } - } - if (Objects.equals(lastestUpdateTime, new Date(0L))) { - lastestUpdateTime = null; - } - return lastestUpdateTime; - } - public List getPolicyDeltas() { return this.policyDeltas; } public void setPolicyDeltas(List policyDeltas) { this.policyDeltas = policyDeltas; } @@ -420,7 +387,7 @@ static public TagPolicies copyHeader(TagPolicies source, String componentService return ret; } - public static ServicePolicies applyDelta(final ServicePolicies servicePolicies, RangerPolicyEngineImpl policyEngine) { + public static ServicePolicies applyDelta(final ServicePolicies servicePolicies, RangerPolicyEngineImpl policyEngine) { ServicePolicies ret = copyHeader(servicePolicies); List oldResourcePolicies = policyEngine.getResourcePolicies(); diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 1edf210162..17a234f534 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -56,11 +56,32 @@ import javax.inject.Inject; import java.io.IOException; import java.io.InputStream; -import java.util.*; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; -import static org.apache.atlas.repository.Constants.*; -import static org.apache.atlas.repository.util.AccessControlUtils.*; +import static org.apache.atlas.repository.Constants.NAME; +import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; +import static org.apache.atlas.repository.Constants.SERVICE_ENTITY_TYPE; +import static org.apache.atlas.repository.Constants.POLICY_ENTITY_TYPE; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_CATEGORY; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_CONNECTION_QN; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_IS_ENABLED; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_PRIORITY; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_SERVICE_NAME; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_SUB_CATEGORY; +import static org.apache.atlas.repository.util.AccessControlUtils.POLICY_CATEGORY_DATAMESH; +import static org.apache.atlas.repository.util.AccessControlUtils.POLICY_CATEGORY_PERSONA; +import static org.apache.atlas.repository.util.AccessControlUtils.POLICY_CATEGORY_PURPOSE; +import static org.apache.atlas.repository.util.AccessControlUtils.getIsPolicyEnabled; +import static org.apache.atlas.repository.util.AccessControlUtils.getPolicyCategory; @Component public class CachePolicyTransformerImpl { @@ -159,24 +180,18 @@ public ServicePolicies getPoliciesDelta(String serviceName, Map tagRangerPolicies = getServicePolicies(tagService, 0); - TagPolicies tagPolicies = new TagPolicies(); tagPolicies.setServiceName(tagServiceName); tagPolicies.setPolicyUpdateTime(new Date()); tagPolicies.setServiceId(tagService.getGuid()); tagPolicies.setPolicyVersion(-1L); - String tagServiceDefName = String.format(RESOURCE_SERVICE_DEF_PATTERN, tagService.getAttribute(NAME)); tagPolicies.setServiceDef(getResourceAsObject(tagServiceDefName, RangerServiceDef.class)); -// tagPolicies.setPolicies(tagRangerPolicies); servicePolicies.setTagPolicies(tagPolicies); -// LOG.info("PolicyDelta: {}: Found tag policies - {}", serviceName, tagRangerPolicies.size()); } } - LOG.info("PolicyDelta: {}: Found {} policies", serviceName, policiesDelta.size()); } diff --git a/intg/src/main/java/org/apache/atlas/ApplicationProperties.java b/intg/src/main/java/org/apache/atlas/ApplicationProperties.java index dc198ba2fa..78b487a8dd 100644 --- a/intg/src/main/java/org/apache/atlas/ApplicationProperties.java +++ b/intg/src/main/java/org/apache/atlas/ApplicationProperties.java @@ -70,7 +70,6 @@ public final class ApplicationProperties extends PropertiesConfiguration { public static final boolean DEFAULT_INDEX_RECOVERY = true; public static final AtlasRunMode DEFAULT_ATLAS_RUN_MODE = AtlasRunMode.PROD; public static final String INDEX_SEARCH_MAX_RESULT_SET_SIZE = "atlas.graph.index.search.max-result-set-size"; - public static final String DELTA_BASED_REFRESH = "atlas.authorizer.enable.delta_based_refresh"; public static final SimpleEntry DB_CACHE_CONF = new SimpleEntry<>("atlas.graph.cache.db-cache", "true"); public static final SimpleEntry DB_CACHE_CLEAN_WAIT_CONF = new SimpleEntry<>("atlas.graph.cache.db-cache-clean-wait", "20"); diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java index c668aa5c90..d50ac8d0ea 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java @@ -52,7 +52,11 @@ import javax.ws.rs.QueryParam; import javax.ws.rs.core.Context; import javax.ws.rs.core.MediaType; -import java.util.*; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Date; import static org.apache.atlas.policytransformer.CachePolicyTransformerImpl.ATTR_SERVICE_LAST_SYNC; import static org.apache.atlas.repository.Constants.PERSONA_ENTITY_TYPE; From 05d3ef3f19fed106d08d570f3f569a164be65646 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Thu, 5 Dec 2024 13:05:36 +0530 Subject: [PATCH 17/31] Map entity purge event to delete change as well --- .../atlas/policytransformer/CachePolicyTransformerImpl.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 17a234f534..db33577349 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -143,6 +143,7 @@ public CachePolicyTransformerImpl(AtlasTypeRegistry typeRegistry) throws AtlasBa this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_CREATE, RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE); this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_UPDATE, RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE); this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_DELETE, RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE); + this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_PURGE, RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE); } public AtlasEntityHeader getService() { @@ -313,14 +314,15 @@ private List getRangerPolicyDelta(AtlasEntityHeader service, // handle delete changes separately as they won't be present in atlas policies List deletedPolicyDeltas = new ArrayList<>(); for (String policyGuid : policyGuids) { - if (policyChanges.get(policyGuid) == EntityAuditActionV2.ENTITY_DELETE) { + int deltaChangeType = auditEventToDeltaChangeType.get(policyChanges.get(policyGuid)); + if (deltaChangeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE) { RangerPolicy deletedPolicy = new RangerPolicy(); deletedPolicy.setGuid(policyGuid); deletedPolicy.setService(serviceName); deletedPolicy.setServiceType(serviceType); RangerPolicyDelta deletedPolicyDelta = new RangerPolicyDelta( deletedPolicy.getId(), - auditEventToDeltaChangeType.get(EntityAuditActionV2.ENTITY_DELETE), + deltaChangeType, deletedPolicy.getVersion(), deletedPolicy ); From f0163ebdb418cabdf3f54c4c53962be90442a1fc Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Fri, 6 Dec 2024 12:31:24 +0530 Subject: [PATCH 18/31] Handler service type for tag policies --- .../CachePolicyTransformerImpl.java | 29 +++++++++++-------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index db33577349..47c9ea21c2 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -160,8 +160,7 @@ public ServicePolicies getPoliciesDelta(String serviceName, Map 0 ? new Date(lastAuditEventTime) : new Date(); -// Date policyUpdateTime = new Date(); + Date policyUpdateTime = lastAuditEventTime > 0 ? new Date(lastAuditEventTime) : new Date(); servicePolicies.setPolicyUpdateTime(policyUpdateTime); if (service != null) { @@ -171,29 +170,35 @@ public ServicePolicies getPoliciesDelta(String serviceName, Map policiesDelta = getRangerPolicyDelta(service, policyChanges); - servicePolicies.setPolicyDeltas(policiesDelta); + ArrayList policyGuids = new ArrayList<>(policyChanges.keySet()); + List allAtlasPolicies = getAtlasPolicies(serviceName, POLICY_BATCH_SIZE, policyGuids); + List atlasServicePolicies = allAtlasPolicies.stream().filter(x -> serviceName.equals(x.getAttribute(ATTR_POLICY_SERVICE_NAME))).collect(Collectors.toList()); + List policiesDelta = getRangerPolicyDelta(service, policyChanges, atlasServicePolicies); - //Process tag based policies + // Process tag based policies String tagServiceName = (String) service.getAttribute(ATTR_SERVICE_TAG_SERVICE); if (StringUtils.isNotEmpty(tagServiceName)) { AtlasEntityHeader tagService = getServiceEntity(tagServiceName); - if (tagService != null) { TagPolicies tagPolicies = new TagPolicies(); - tagPolicies.setServiceName(tagServiceName); tagPolicies.setPolicyUpdateTime(new Date()); tagPolicies.setServiceId(tagService.getGuid()); tagPolicies.setPolicyVersion(-1L); + String tagServiceDefName = String.format(RESOURCE_SERVICE_DEF_PATTERN, tagService.getAttribute(NAME)); tagPolicies.setServiceDef(getResourceAsObject(tagServiceDefName, RangerServiceDef.class)); - servicePolicies.setTagPolicies(tagPolicies); + + List tagServicePolicies = allAtlasPolicies.stream().filter(x -> tagServiceName.equals(x.getAttribute(ATTR_POLICY_SERVICE_NAME))).collect(Collectors.toList()); + List tagPoliciesDelta = getRangerPolicyDelta(tagService, policyChanges, tagServicePolicies); + policiesDelta.addAll(tagPoliciesDelta); } } - LOG.info("PolicyDelta: {}: Found {} policies", serviceName, policiesDelta.size()); + servicePolicies.setPolicyDeltas(policiesDelta); + + LOG.info("PolicyDelta: {}: Found total delta={}", serviceName, policiesDelta.size()); } } catch (Exception e) { @@ -287,7 +292,7 @@ private List getServicePolicies(AtlasEntityHeader service, int bat return servicePolicies; } - private List getRangerPolicyDelta(AtlasEntityHeader service, Map policyChanges) throws AtlasBaseException, IOException { + private List getRangerPolicyDelta(AtlasEntityHeader service, Map policyChanges, List atlasPolicies) throws AtlasBaseException, IOException { String serviceName = (String) service.getAttribute("name"); String serviceType = (String) service.getAttribute("authServiceType"); AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getRangerPolicyDelta." + serviceName); @@ -298,7 +303,7 @@ private List getRangerPolicyDelta(AtlasEntityHeader service, } ArrayList policyGuids = new ArrayList<>(policyChanges.keySet()); - List atlasPolicies = getAtlasPolicies(serviceName, POLICY_BATCH_SIZE, policyGuids); +// List atlasPolicies = getAtlasPolicies(serviceName, POLICY_BATCH_SIZE, policyGuids); List rangerPolicies = new ArrayList<>(); if (CollectionUtils.isNotEmpty(atlasPolicies)) { @@ -340,7 +345,7 @@ private List getRangerPolicyDelta(AtlasEntityHeader service, } public Map createPolicyChangeMap(String serviceName, List events) { - Map policyChanges = new HashMap<>(); + Map policyChanges = new HashMap<>(); for (EntityAuditEventV2 event : events) { if (POLICY_ENTITY_TYPE.equals(event.getTypeName()) && !policyChanges.containsKey(event.getEntityId())) { policyChanges.put(event.getEntityId(), event.getAction()); From f41b7c8a77c289da42f6b4e4d791ec5306c563a3 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Fri, 6 Dec 2024 13:57:11 +0530 Subject: [PATCH 19/31] Remove commented code --- .../atlas/policytransformer/CachePolicyTransformerImpl.java | 1 - 1 file changed, 1 deletion(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 47c9ea21c2..87df320336 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -303,7 +303,6 @@ private List getRangerPolicyDelta(AtlasEntityHeader service, } ArrayList policyGuids = new ArrayList<>(policyChanges.keySet()); -// List atlasPolicies = getAtlasPolicies(serviceName, POLICY_BATCH_SIZE, policyGuids); List rangerPolicies = new ArrayList<>(); if (CollectionUtils.isNotEmpty(atlasPolicies)) { From dce45ff80046eafa15788bd8bfbb499f5b652f3f Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Mon, 9 Dec 2024 16:57:52 +0530 Subject: [PATCH 20/31] feat: add resources to tag service --- .../atlas-servicedef-atlas_tag.json | 450 +++++++++++++++++- 1 file changed, 448 insertions(+), 2 deletions(-) diff --git a/auth-agents-common/src/main/resources/service-defs/atlas-servicedef-atlas_tag.json b/auth-agents-common/src/main/resources/service-defs/atlas-servicedef-atlas_tag.json index 98ff10877f..cba65d8f6b 100644 --- a/auth-agents-common/src/main/resources/service-defs/atlas-servicedef-atlas_tag.json +++ b/auth-agents-common/src/main/resources/service-defs/atlas-servicedef-atlas_tag.json @@ -29,11 +29,457 @@ "uiHint":"{ \"singleValue\":true }", "label": "TAG", "description": "TAG" + }, + { + "itemId": 2, + "name": "type-category", + "type": "string", + "level": 10, + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "true" + }, + "label": "Type Catagory", + "description": "Type Catagory" + }, + { + "itemId": 3, + "name": "type", + "type": "string", + "level": 20, + "mandatory": true, + "parent": "type-category", + "isValidLeaf": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "false" + }, + "label": "Type Name", + "description": "Type Name", + "accessTypeRestrictions": ["type-read" ,"type-create", "type-update", "type-delete" ] + }, + { + "itemId": 4, + "name": "entity-type", + "type": "string", + "level": 10, + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "false" + }, + "label": "Entity Type", + "description": "Entity Type" + }, + { + "itemId": 5, + "name": "entity-classification", + "type": "string", + "level": 20, + "mandatory": true, + "parent": "entity-type", + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "false" + }, + "label": "Entity Classification", + "description": "Entity Classification" + }, + { + "itemId": 6, + "name": "entity", + "type": "string", + "level": 30, + "mandatory": true, + "parent": "entity-classification", + "isValidLeaf": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "true" + }, + "label": "Entity ID", + "description": "Entity ID", + "accessTypeRestrictions": ["entity-read", "entity-create", "entity-update", "entity-delete"] + }, + { + "itemId": 7, + "name": "atlas-service", + "type": "string", + "level": 10, + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "true" + }, + "label": "Atlas Service", + "description": "Atlas Service", + "accessTypeRestrictions": ["admin-import", "admin-export", "admin-purge", "admin-audits", "admin-entity-audits", "admin-repair-index", "admin-task-cud"] + }, + { + "itemId": 8, + "name": "relationship-type", + "type": "string", + "level": 10, + "mandatory": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "false" + }, + "label": "Relationship Type", + "description": "Relationship Type" + }, + { + "itemId": 9, + "name": "end-one-entity-type", + "type": "string", + "level": 20, + "mandatory": true, + "parent": "relationship-type", + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "false" + }, + "label": "End1 Entity Type", + "description": "End1 Entity Type" + }, + { + "itemId": 10, + "name": "end-one-entity-classification", + "type": "string", + "level": 30, + "mandatory": true, + "parent": "end-one-entity-type", + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "false" + }, + "label": "End1 Entity Classification", + "description": "End1 Entity Classification" + }, + { + "itemId": 11, + "name": "end-one-entity", + "type": "string", + "level": 40, + "mandatory": true, + "parent": "end-one-entity-classification", + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "true" + }, + "label": "End1 Entity ID", + "description": "End1 Entity ID" + }, + { + "itemId": 12, + "name": "end-two-entity-type", + "type": "string", + "level": 50, + "mandatory": true, + "parent": "end-one-entity", + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "false" + }, + "label": "End2 Entity Type", + "description": "End2 Entity Type" + }, + { + "itemId": 13, + "name": "end-two-entity-classification", + "type": "string", + "level": 60, + "mandatory": true, + "parent": "end-two-entity-type", + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "false" + }, + "label": "End2 Entity Classification", + "description": "End2 Entity Classification" + }, + { + "itemId": 14, + "name": "end-two-entity", + "type": "string", + "level": 70, + "mandatory": true, + "parent": "end-two-entity-classification", + "isValidLeaf": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "true" + }, + "label": "End2 Entity ID", + "description": "End2 Entity ID", + "accessTypeRestrictions": [ + "add-relationship", + "update-relationship", + "remove-relationship" + ] + }, + { + "itemId": 15, + "name": "entity-label", + "type": "string", + "level": 40, + "mandatory": true, + "parent": "entity", + "isValidLeaf": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "true" + }, + "label": "Label", + "description": "Label", + "accessTypeRestrictions": [ + "entity-add-label", + "entity-remove-label" + ] + }, + { + "itemId": 16, + "name": "entity-business-metadata", + "type": "string", + "level": 40, + "mandatory": true, + "parent": "entity", + "isValidLeaf": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "true" + }, + "label": "Business Metadata", + "description": "Business Metadata", + "accessTypeRestrictions": [ + "entity-update-business-metadata" + ] + }, + { + "itemId": 17, + "name": "classification", + "type": "string", + "level": 40, + "mandatory": true, + "parent": "entity", + "isValidLeaf": true, + "lookupSupported": true, + "recursiveSupported": false, + "excludesSupported": true, + "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", + "matcherOptions": { + "wildCard": "true", + "ignoreCase": "false" + }, + "label": "Targetted classifications", + "description": "Targetted classifications", + "accessTypeRestrictions": [ + "entity-add-classification", + "entity-update-classification", + "entity-remove-classification" + ] } ], - "accessTypes": - [ + "accessTypes": [ + { + "itemId": 1, + "name": "type-create", + "label": "Create Type", + "impliedGrants": + [ + "type-read" + ] + }, + { + "itemId": 2, + "name": "type-update", + "label": "Update Type", + "impliedGrants": + [ + "type-read" + ] + }, + { + "itemId": 3, + "name": "type-delete", + "label": "Delete Type", + "impliedGrants": + [ + "type-read" + ] + }, + { + "itemId": 4, + "name": "entity-read", + "label": "Read Entity" + }, + { + "itemId": 5, + "name": "entity-create", + "label": "Create Entity" + }, + { + "itemId": 6, + "name": "entity-update", + "label": "Update Entity" + }, + { + "itemId": 7, + "name": "entity-delete", + "label": "Delete Entity" + }, + { + "itemId": 8, + "name": "entity-add-classification", + "label": "Add Classification" + }, + { + "itemId": 9, + "name": "entity-update-classification", + "label": "Update Classification" + }, + { + "itemId": 10, + "name": "entity-remove-classification", + "label": "Remove Classification" + }, + { + "itemId": 11, + "name": "admin-export", + "label": "Admin Export" + }, + { + "itemId": 12, + "name": "admin-import", + "label": "Admin Import" + }, + { + "itemId": 13, + "name": "add-relationship", + "label": "Add Relationship" + }, + { + "itemId": 14, + "name": "update-relationship", + "label": "Update Relationship" + }, + { + "itemId": 15, + "name": "remove-relationship", + "label": "Remove Relationship" + }, + { + "itemId": 16, + "name": "admin-purge", + "label": "Admin Purge" + }, + { + "itemId": 17, + "name": "entity-add-label", + "label": "Add Label" + }, + { + "itemId": 18, + "name": "entity-remove-label", + "label": "Remove Label" + }, + { + "itemId": 19, + "name": "entity-update-business-metadata", + "label": "Update Business Metadata" + }, + { + "itemId": 20, + "name": "type-read", + "label": "Read Type" + }, + { + "itemId": 21, + "name": "admin-audits", + "label": "Admin Audits" + }, + { + "itemId": 22, + "name": "admin-entity-audits", + "label": "Admin Entity Audits" + }, + { + "itemId": 23, + "name": "admin-repair-index", + "label": "Admin Repair Index" + }, + { + "itemId": 24, + "name": "admin-task-cud", + "label": "Admin task CUD API" + }, + { + "itemId": 25, + "name": "admin-featureFlag-cud", + "label": "Admin featureflag CUD API" + } ], From 9d71641cdf724087c63e4d18790cc734ae46fe0b Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Mon, 9 Dec 2024 18:21:41 +0530 Subject: [PATCH 21/31] Revert "feat: add resources to tag service" This reverts commit dce45ff80046eafa15788bd8bfbb499f5b652f3f. --- .../atlas-servicedef-atlas_tag.json | 450 +----------------- 1 file changed, 2 insertions(+), 448 deletions(-) diff --git a/auth-agents-common/src/main/resources/service-defs/atlas-servicedef-atlas_tag.json b/auth-agents-common/src/main/resources/service-defs/atlas-servicedef-atlas_tag.json index cba65d8f6b..98ff10877f 100644 --- a/auth-agents-common/src/main/resources/service-defs/atlas-servicedef-atlas_tag.json +++ b/auth-agents-common/src/main/resources/service-defs/atlas-servicedef-atlas_tag.json @@ -29,457 +29,11 @@ "uiHint":"{ \"singleValue\":true }", "label": "TAG", "description": "TAG" - }, - { - "itemId": 2, - "name": "type-category", - "type": "string", - "level": 10, - "mandatory": true, - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "true" - }, - "label": "Type Catagory", - "description": "Type Catagory" - }, - { - "itemId": 3, - "name": "type", - "type": "string", - "level": 20, - "mandatory": true, - "parent": "type-category", - "isValidLeaf": true, - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "false" - }, - "label": "Type Name", - "description": "Type Name", - "accessTypeRestrictions": ["type-read" ,"type-create", "type-update", "type-delete" ] - }, - { - "itemId": 4, - "name": "entity-type", - "type": "string", - "level": 10, - "mandatory": true, - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "false" - }, - "label": "Entity Type", - "description": "Entity Type" - }, - { - "itemId": 5, - "name": "entity-classification", - "type": "string", - "level": 20, - "mandatory": true, - "parent": "entity-type", - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "false" - }, - "label": "Entity Classification", - "description": "Entity Classification" - }, - { - "itemId": 6, - "name": "entity", - "type": "string", - "level": 30, - "mandatory": true, - "parent": "entity-classification", - "isValidLeaf": true, - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "true" - }, - "label": "Entity ID", - "description": "Entity ID", - "accessTypeRestrictions": ["entity-read", "entity-create", "entity-update", "entity-delete"] - }, - { - "itemId": 7, - "name": "atlas-service", - "type": "string", - "level": 10, - "mandatory": true, - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "true" - }, - "label": "Atlas Service", - "description": "Atlas Service", - "accessTypeRestrictions": ["admin-import", "admin-export", "admin-purge", "admin-audits", "admin-entity-audits", "admin-repair-index", "admin-task-cud"] - }, - { - "itemId": 8, - "name": "relationship-type", - "type": "string", - "level": 10, - "mandatory": true, - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "false" - }, - "label": "Relationship Type", - "description": "Relationship Type" - }, - { - "itemId": 9, - "name": "end-one-entity-type", - "type": "string", - "level": 20, - "mandatory": true, - "parent": "relationship-type", - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "false" - }, - "label": "End1 Entity Type", - "description": "End1 Entity Type" - }, - { - "itemId": 10, - "name": "end-one-entity-classification", - "type": "string", - "level": 30, - "mandatory": true, - "parent": "end-one-entity-type", - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "false" - }, - "label": "End1 Entity Classification", - "description": "End1 Entity Classification" - }, - { - "itemId": 11, - "name": "end-one-entity", - "type": "string", - "level": 40, - "mandatory": true, - "parent": "end-one-entity-classification", - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "true" - }, - "label": "End1 Entity ID", - "description": "End1 Entity ID" - }, - { - "itemId": 12, - "name": "end-two-entity-type", - "type": "string", - "level": 50, - "mandatory": true, - "parent": "end-one-entity", - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "false" - }, - "label": "End2 Entity Type", - "description": "End2 Entity Type" - }, - { - "itemId": 13, - "name": "end-two-entity-classification", - "type": "string", - "level": 60, - "mandatory": true, - "parent": "end-two-entity-type", - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "false" - }, - "label": "End2 Entity Classification", - "description": "End2 Entity Classification" - }, - { - "itemId": 14, - "name": "end-two-entity", - "type": "string", - "level": 70, - "mandatory": true, - "parent": "end-two-entity-classification", - "isValidLeaf": true, - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "true" - }, - "label": "End2 Entity ID", - "description": "End2 Entity ID", - "accessTypeRestrictions": [ - "add-relationship", - "update-relationship", - "remove-relationship" - ] - }, - { - "itemId": 15, - "name": "entity-label", - "type": "string", - "level": 40, - "mandatory": true, - "parent": "entity", - "isValidLeaf": true, - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "true" - }, - "label": "Label", - "description": "Label", - "accessTypeRestrictions": [ - "entity-add-label", - "entity-remove-label" - ] - }, - { - "itemId": 16, - "name": "entity-business-metadata", - "type": "string", - "level": 40, - "mandatory": true, - "parent": "entity", - "isValidLeaf": true, - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "true" - }, - "label": "Business Metadata", - "description": "Business Metadata", - "accessTypeRestrictions": [ - "entity-update-business-metadata" - ] - }, - { - "itemId": 17, - "name": "classification", - "type": "string", - "level": 40, - "mandatory": true, - "parent": "entity", - "isValidLeaf": true, - "lookupSupported": true, - "recursiveSupported": false, - "excludesSupported": true, - "matcher": "org.apache.atlas.plugin.resourcematcher.RangerDefaultResourceMatcher", - "matcherOptions": { - "wildCard": "true", - "ignoreCase": "false" - }, - "label": "Targetted classifications", - "description": "Targetted classifications", - "accessTypeRestrictions": [ - "entity-add-classification", - "entity-update-classification", - "entity-remove-classification" - ] } ], - "accessTypes": [ - { - "itemId": 1, - "name": "type-create", - "label": "Create Type", - "impliedGrants": - [ - "type-read" - ] - }, - { - "itemId": 2, - "name": "type-update", - "label": "Update Type", - "impliedGrants": - [ - "type-read" - ] - }, - { - "itemId": 3, - "name": "type-delete", - "label": "Delete Type", - "impliedGrants": - [ - "type-read" - ] - }, - { - "itemId": 4, - "name": "entity-read", - "label": "Read Entity" - }, - { - "itemId": 5, - "name": "entity-create", - "label": "Create Entity" - }, - { - "itemId": 6, - "name": "entity-update", - "label": "Update Entity" - }, - { - "itemId": 7, - "name": "entity-delete", - "label": "Delete Entity" - }, - { - "itemId": 8, - "name": "entity-add-classification", - "label": "Add Classification" - }, - { - "itemId": 9, - "name": "entity-update-classification", - "label": "Update Classification" - }, - { - "itemId": 10, - "name": "entity-remove-classification", - "label": "Remove Classification" - }, - { - "itemId": 11, - "name": "admin-export", - "label": "Admin Export" - }, - { - "itemId": 12, - "name": "admin-import", - "label": "Admin Import" - }, - { - "itemId": 13, - "name": "add-relationship", - "label": "Add Relationship" - }, - { - "itemId": 14, - "name": "update-relationship", - "label": "Update Relationship" - }, - { - "itemId": 15, - "name": "remove-relationship", - "label": "Remove Relationship" - }, - { - "itemId": 16, - "name": "admin-purge", - "label": "Admin Purge" - }, - { - "itemId": 17, - "name": "entity-add-label", - "label": "Add Label" - }, - { - "itemId": 18, - "name": "entity-remove-label", - "label": "Remove Label" - }, - { - "itemId": 19, - "name": "entity-update-business-metadata", - "label": "Update Business Metadata" - }, - { - "itemId": 20, - "name": "type-read", - "label": "Read Type" - }, - { - "itemId": 21, - "name": "admin-audits", - "label": "Admin Audits" - }, - { - "itemId": 22, - "name": "admin-entity-audits", - "label": "Admin Entity Audits" - }, - { - "itemId": 23, - "name": "admin-repair-index", - "label": "Admin Repair Index" - }, - { - "itemId": 24, - "name": "admin-task-cud", - "label": "Admin task CUD API" - }, - { - "itemId": 25, - "name": "admin-featureFlag-cud", - "label": "Admin featureflag CUD API" - } + "accessTypes": + [ ], From 8b9f30b89aa13277629a7d874d30f712575941c9 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Mon, 9 Dec 2024 18:22:13 +0530 Subject: [PATCH 22/31] feat: add atlas authZ in case of relationship evaluator --- .../policytransformer/CachePolicyTransformerImpl.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 87df320336..bfab5baa6c 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -82,6 +82,7 @@ import static org.apache.atlas.repository.util.AccessControlUtils.POLICY_CATEGORY_PURPOSE; import static org.apache.atlas.repository.util.AccessControlUtils.getIsPolicyEnabled; import static org.apache.atlas.repository.util.AccessControlUtils.getPolicyCategory; +import static org.apache.atlas.services.tag.RangerServiceTag.TAG_RESOURCE_NAME; @Component public class CachePolicyTransformerImpl { @@ -709,7 +710,14 @@ private RangerPolicy getRangerPolicy(AtlasEntityHeader atlasPolicy, String servi //policy.setId(atlasPolicy.getGuid()); policy.setName((String) atlasPolicy.getAttribute(QUALIFIED_NAME)); policy.setService((String) atlasPolicy.getAttribute(ATTR_POLICY_SERVICE_NAME)); - policy.setServiceType(serviceType); + + // Adding atlas as serviceType for tag policies, as atlas_tag doesn't have all the resource available for evaluation + if (serviceType != null && serviceType.equals(TAG_RESOURCE_NAME) && policy.getService().equals("atlas")) { + policy.setServiceType("atlas"); + } else { + policy.setServiceType(serviceType); + } + policy.setGuid(atlasPolicy.getGuid()); policy.setCreatedBy(atlasPolicy.getCreatedBy()); policy.setCreateTime(atlasPolicy.getCreateTime()); From d5218aeeea92e7a78ccd2189b9aaafd1a8fe9026 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Tue, 10 Dec 2024 10:14:48 +0530 Subject: [PATCH 23/31] Remove all updated policies and take new as created --- .../org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java index 0c8267b5c7..08f5498c65 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java @@ -72,7 +72,7 @@ public static List applyDeltas(List policies, List(policies); for (RangerPolicyDelta delta : deltas) { - if (!serviceType.equals(delta.getServiceType())) { + if (!serviceType.equals(delta.getPolicyServiceName())) { continue; } @@ -92,7 +92,7 @@ public static List applyDeltas(List policies, List applyDeltas(List policies, List 1) { + if (CollectionUtils.isEmpty(deletedPolicies)) { LOG.warn("Unexpected: found no policy or multiple policies for CHANGE_TYPE_POLICY_UPDATE: " + Arrays.toString(deletedPolicies.toArray())); } break; From 6021ef38a0d8868f8ddf57c1e92f4f2480cfc0d8 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Tue, 10 Dec 2024 10:26:01 +0530 Subject: [PATCH 24/31] Fix missing method left over changes --- .../org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java index 08f5498c65..340dcbec15 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java @@ -72,7 +72,7 @@ public static List applyDeltas(List policies, List(policies); for (RangerPolicyDelta delta : deltas) { - if (!serviceType.equals(delta.getPolicyServiceName())) { + if (!serviceType.equals(delta.getServiceType())) { continue; } From 4bf4f761dfa824493f897ea6a1151147f786ed50 Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Tue, 10 Dec 2024 11:10:36 +0530 Subject: [PATCH 25/31] Use service name to separate tag policies during delta apply --- .../atlas/plugin/model/RangerPolicyDelta.java | 3 +++ .../plugin/store/EmbeddedServiceDefsUtil.java | 1 + .../plugin/util/RangerPolicyDeltaUtil.java | 24 +++++++++---------- .../atlas/plugin/util/ServicePolicies.java | 4 ++-- .../CachePolicyTransformerImpl.java | 14 +++++------ 5 files changed, 25 insertions(+), 21 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java index b5f9b58233..f7dcba5466 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java @@ -89,6 +89,9 @@ public RangerPolicyDelta(final Long id, final Integer changeType, final Long pol public void setPolicy(RangerPolicy policy) { this.policy = policy; } + @JsonIgnore + public String getPolicyServiceName() { return policy != null ? policy.getService() : null;} + @Override public String toString() { return "id:" + id diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/EmbeddedServiceDefsUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/EmbeddedServiceDefsUtil.java index 270141f22f..630a56819e 100755 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/EmbeddedServiceDefsUtil.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/EmbeddedServiceDefsUtil.java @@ -53,6 +53,7 @@ public class EmbeddedServiceDefsUtil { private static final String PROPERTY_SUPPORTED_SERVICE_DEFS = "ranger.supportedcomponents"; private Set supportedServiceDefs; public static final String EMBEDDED_SERVICEDEF_TAG_NAME = "tag"; + public static final String EMBEDDED_SERVICEDEF_TAG_SERVICE_NAME = "atlas_tag"; public static final String EMBEDDED_SERVICEDEF_HDFS_NAME = "hdfs"; public static final String EMBEDDED_SERVICEDEF_HBASE_NAME = "hbase"; public static final String EMBEDDED_SERVICEDEF_HIVE_NAME = "hive"; diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java index 340dcbec15..2e41298a96 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java @@ -39,9 +39,9 @@ public class RangerPolicyDeltaUtil { private static final Log PERF_POLICY_DELTA_LOG = RangerPerfTracer.getPerfLogger("policy.delta"); - public static List applyDeltas(List policies, List deltas, String serviceType) { + public static List applyDeltas(List policies, List deltas, String serviceName) { if (LOG.isDebugEnabled()) { - LOG.debug("==> applyDeltas(serviceType=" + serviceType + ")"); + LOG.debug("==> applyDeltas(serviceType=" + serviceName + ")"); } List ret; @@ -52,27 +52,27 @@ public static List applyDeltas(List policies, List(policies); for (RangerPolicyDelta delta : deltas) { - if (!serviceType.equals(delta.getServiceType())) { + if (!serviceName.equals(delta.getPolicyServiceName())) { continue; } @@ -132,7 +132,7 @@ public static List applyDeltas(List policies, List applyDeltas(List policies, List oldResourcePolicies = policyEngine.getResourcePolicies(); List oldTagPolicies = policyEngine.getTagPolicies(); - List newResourcePolicies = RangerPolicyDeltaUtil.applyDeltas(oldResourcePolicies, servicePolicies.getPolicyDeltas(), servicePolicies.getServiceDef().getName()); + List newResourcePolicies = RangerPolicyDeltaUtil.applyDeltas(oldResourcePolicies, servicePolicies.getPolicyDeltas(), servicePolicies.getServiceName()); ret.setPolicies(newResourcePolicies); @@ -402,7 +402,7 @@ public static ServicePolicies applyDelta(final ServicePolicies servicePolicies, if (LOG.isDebugEnabled()) { LOG.debug("applyingDeltas for tag policies"); } - newTagPolicies = RangerPolicyDeltaUtil.applyDeltas(oldTagPolicies, servicePolicies.getPolicyDeltas(), servicePolicies.getTagPolicies().getServiceDef().getName()); + newTagPolicies = RangerPolicyDeltaUtil.applyDeltas(oldTagPolicies, servicePolicies.getPolicyDeltas(), servicePolicies.getTagPolicies().getServiceName()); } else { if (LOG.isDebugEnabled()) { LOG.debug("No need to apply deltas for tag policies"); diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index bfab5baa6c..b3b1225ca4 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -710,13 +710,13 @@ private RangerPolicy getRangerPolicy(AtlasEntityHeader atlasPolicy, String servi //policy.setId(atlasPolicy.getGuid()); policy.setName((String) atlasPolicy.getAttribute(QUALIFIED_NAME)); policy.setService((String) atlasPolicy.getAttribute(ATTR_POLICY_SERVICE_NAME)); - - // Adding atlas as serviceType for tag policies, as atlas_tag doesn't have all the resource available for evaluation - if (serviceType != null && serviceType.equals(TAG_RESOURCE_NAME) && policy.getService().equals("atlas")) { - policy.setServiceType("atlas"); - } else { - policy.setServiceType(serviceType); - } + policy.setServiceType(serviceType); +// // Adding atlas as serviceType for tag policies, as atlas_tag doesn't have all the resource available for evaluation +// if (serviceType != null && serviceType.equals(TAG_RESOURCE_NAME) && policy.getService().equals("atlas")) { +// policy.setServiceType("atlas"); +// } else { +// policy.setServiceType(serviceType); +// } policy.setGuid(atlasPolicy.getGuid()); policy.setCreatedBy(atlasPolicy.getCreatedBy()); From ed4a51ec958ffe4883acf626d09a9f69c1bf0a52 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Tue, 10 Dec 2024 11:19:41 +0530 Subject: [PATCH 26/31] Revert "Use service name to separate tag policies during delta apply" This reverts commit 4bf4f761dfa824493f897ea6a1151147f786ed50. --- .../atlas/plugin/model/RangerPolicyDelta.java | 3 --- .../plugin/store/EmbeddedServiceDefsUtil.java | 1 - .../plugin/util/RangerPolicyDeltaUtil.java | 24 +++++++++---------- .../atlas/plugin/util/ServicePolicies.java | 4 ++-- .../CachePolicyTransformerImpl.java | 14 +++++------ 5 files changed, 21 insertions(+), 25 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java index f7dcba5466..b5f9b58233 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java @@ -89,9 +89,6 @@ public RangerPolicyDelta(final Long id, final Integer changeType, final Long pol public void setPolicy(RangerPolicy policy) { this.policy = policy; } - @JsonIgnore - public String getPolicyServiceName() { return policy != null ? policy.getService() : null;} - @Override public String toString() { return "id:" + id diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/EmbeddedServiceDefsUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/EmbeddedServiceDefsUtil.java index 630a56819e..270141f22f 100755 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/EmbeddedServiceDefsUtil.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/store/EmbeddedServiceDefsUtil.java @@ -53,7 +53,6 @@ public class EmbeddedServiceDefsUtil { private static final String PROPERTY_SUPPORTED_SERVICE_DEFS = "ranger.supportedcomponents"; private Set supportedServiceDefs; public static final String EMBEDDED_SERVICEDEF_TAG_NAME = "tag"; - public static final String EMBEDDED_SERVICEDEF_TAG_SERVICE_NAME = "atlas_tag"; public static final String EMBEDDED_SERVICEDEF_HDFS_NAME = "hdfs"; public static final String EMBEDDED_SERVICEDEF_HBASE_NAME = "hbase"; public static final String EMBEDDED_SERVICEDEF_HIVE_NAME = "hive"; diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java index 2e41298a96..340dcbec15 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java @@ -39,9 +39,9 @@ public class RangerPolicyDeltaUtil { private static final Log PERF_POLICY_DELTA_LOG = RangerPerfTracer.getPerfLogger("policy.delta"); - public static List applyDeltas(List policies, List deltas, String serviceName) { + public static List applyDeltas(List policies, List deltas, String serviceType) { if (LOG.isDebugEnabled()) { - LOG.debug("==> applyDeltas(serviceType=" + serviceName + ")"); + LOG.debug("==> applyDeltas(serviceType=" + serviceType + ")"); } List ret; @@ -52,27 +52,27 @@ public static List applyDeltas(List policies, List(policies); for (RangerPolicyDelta delta : deltas) { - if (!serviceName.equals(delta.getPolicyServiceName())) { + if (!serviceType.equals(delta.getServiceType())) { continue; } @@ -132,7 +132,7 @@ public static List applyDeltas(List policies, List applyDeltas(List policies, List oldResourcePolicies = policyEngine.getResourcePolicies(); List oldTagPolicies = policyEngine.getTagPolicies(); - List newResourcePolicies = RangerPolicyDeltaUtil.applyDeltas(oldResourcePolicies, servicePolicies.getPolicyDeltas(), servicePolicies.getServiceName()); + List newResourcePolicies = RangerPolicyDeltaUtil.applyDeltas(oldResourcePolicies, servicePolicies.getPolicyDeltas(), servicePolicies.getServiceDef().getName()); ret.setPolicies(newResourcePolicies); @@ -402,7 +402,7 @@ public static ServicePolicies applyDelta(final ServicePolicies servicePolicies, if (LOG.isDebugEnabled()) { LOG.debug("applyingDeltas for tag policies"); } - newTagPolicies = RangerPolicyDeltaUtil.applyDeltas(oldTagPolicies, servicePolicies.getPolicyDeltas(), servicePolicies.getTagPolicies().getServiceName()); + newTagPolicies = RangerPolicyDeltaUtil.applyDeltas(oldTagPolicies, servicePolicies.getPolicyDeltas(), servicePolicies.getTagPolicies().getServiceDef().getName()); } else { if (LOG.isDebugEnabled()) { LOG.debug("No need to apply deltas for tag policies"); diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index b3b1225ca4..bfab5baa6c 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -710,13 +710,13 @@ private RangerPolicy getRangerPolicy(AtlasEntityHeader atlasPolicy, String servi //policy.setId(atlasPolicy.getGuid()); policy.setName((String) atlasPolicy.getAttribute(QUALIFIED_NAME)); policy.setService((String) atlasPolicy.getAttribute(ATTR_POLICY_SERVICE_NAME)); - policy.setServiceType(serviceType); -// // Adding atlas as serviceType for tag policies, as atlas_tag doesn't have all the resource available for evaluation -// if (serviceType != null && serviceType.equals(TAG_RESOURCE_NAME) && policy.getService().equals("atlas")) { -// policy.setServiceType("atlas"); -// } else { -// policy.setServiceType(serviceType); -// } + + // Adding atlas as serviceType for tag policies, as atlas_tag doesn't have all the resource available for evaluation + if (serviceType != null && serviceType.equals(TAG_RESOURCE_NAME) && policy.getService().equals("atlas")) { + policy.setServiceType("atlas"); + } else { + policy.setServiceType(serviceType); + } policy.setGuid(atlasPolicy.getGuid()); policy.setCreatedBy(atlasPolicy.getCreatedBy()); From 8df0b5e74788b01f076a043000ba1c3b64fb6d74 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Tue, 10 Dec 2024 14:34:52 +0530 Subject: [PATCH 27/31] feat: update policy delta logic full revamp --- .../atlas/plugin/model/RangerPolicy.java | 5 +- .../policyengine/RangerPolicyRepository.java | 1 - .../plugin/util/RangerPolicyDeltaUtil.java | 129 +++++------------- .../atlas/plugin/util/ServicePolicies.java | 55 +++----- .../CachePolicyTransformerImpl.java | 66 +++++---- 5 files changed, 96 insertions(+), 160 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java index 3a4e52fd3b..e071531f80 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java @@ -532,10 +532,7 @@ public void setIsDenyAllElse(Boolean isDenyAllElse) { @JsonIgnore public String getAtlasGuid() { - if (getGuid().length() > 36) { - return getGuid().substring(0, 36); - } - return getGuid(); + return getGuid().length() > 36 ? getGuid().substring(0, 36) : getGuid(); } @Override diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyRepository.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyRepository.java index 4324ccb2cd..fc2112f8fc 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyRepository.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyRepository.java @@ -1355,7 +1355,6 @@ private void deletePolicyEvaluator(RangerPolicyEvaluator evaluator) { } private RangerPolicyEvaluator update(final RangerPolicyDelta delta, final RangerPolicyEvaluator currentEvaluator) { - LOG.info("PolicyDelta: RangerPolicyRepository.update is called, policyGuid: "+delta.getPolicyGuid()); if (LOG.isDebugEnabled()) { LOG.debug("==> RangerPolicyRepository.update(delta=" + delta + ", currentEvaluator=" + (currentEvaluator == null ? null : currentEvaluator.getPolicy()) + ")"); } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java index 340dcbec15..a37187aee8 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java @@ -30,8 +30,8 @@ import java.util.ArrayList; import java.util.Arrays; -import java.util.Iterator; import java.util.List; +import java.util.Map; public class RangerPolicyDeltaUtil { @@ -39,116 +39,59 @@ public class RangerPolicyDeltaUtil { private static final Log PERF_POLICY_DELTA_LOG = RangerPerfTracer.getPerfLogger("policy.delta"); + public static List deletePoliciesByDelta(List policies, Map deltas) { + if (MapUtils.isNotEmpty(deltas)) { + List ret = new ArrayList<>(policies); + ret.removeIf(policy -> deltas.containsKey(policy.getAtlasGuid())); + return ret; + } else { + return policies; + } + } + public static List applyDeltas(List policies, List deltas, String serviceType) { if (LOG.isDebugEnabled()) { LOG.debug("==> applyDeltas(serviceType=" + serviceType + ")"); } - List ret; - - RangerPerfTracer perf = null; - - if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICY_DELTA_LOG)) { - perf = RangerPerfTracer.getPerfTracer(PERF_POLICY_DELTA_LOG, "RangerPolicyDelta.applyDeltas()"); + if (CollectionUtils.isEmpty(deltas)) { + if (LOG.isDebugEnabled()) { + LOG.debug("applyDeltas called with empty deltas. Returning policies without changes."); + } + return policies; } - boolean hasExpectedServiceType = false; + boolean hasExpectedServiceType = deltas.stream().anyMatch(delta -> serviceType.equals(delta.getServiceType())); - if (CollectionUtils.isNotEmpty(deltas)) { + if (!hasExpectedServiceType) { if (LOG.isDebugEnabled()) { - LOG.debug("applyDeltas(deltas=" + Arrays.toString(deltas.toArray()) + ", serviceType=" + serviceType + ")"); - } - - for (RangerPolicyDelta delta : deltas) { - if (serviceType.equals(delta.getServiceType())) { - hasExpectedServiceType = true; - break; - } else if (!serviceType.equals(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME) && !delta.getServiceType().equals(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME)) { - LOG.warn("Found unexpected serviceType in policyDelta:[" + delta + "]. Was expecting serviceType:[" + serviceType + "]. Should NOT have come here!! Ignoring delta and continuing"); - } + LOG.debug("No deltas match the expected serviceType: " + serviceType); } + return policies; + } - if (hasExpectedServiceType) { - ret = new ArrayList<>(policies); + List updatedPolicies = new ArrayList<>(policies); - for (RangerPolicyDelta delta : deltas) { - if (!serviceType.equals(delta.getServiceType())) { - continue; - } - - int changeType = delta.getChangeType(); - - if (changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE || changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE || changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE) { - String policyId = delta.getPolicyGuid(); // change to getGuid() as id is not set in policy - - if (policyId == null) { - continue; - } - - List deletedPolicies = new ArrayList<>(); - - Iterator iter = ret.iterator(); - - while (iter.hasNext()) { - RangerPolicy policy = iter.next(); - if ( - (changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE && policyId.equals(policy.getAtlasGuid())) || - (changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE && policyId.equals(policy.getAtlasGuid())) - ){ - deletedPolicies.add(policy); - iter.remove(); - } - } - - switch(changeType) { - case RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE: { - if (CollectionUtils.isNotEmpty(deletedPolicies)) { - LOG.warn("Unexpected: found existing policy for CHANGE_TYPE_POLICY_CREATE: " + Arrays.toString(deletedPolicies.toArray())); - } - break; - } - case RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE: { - if (CollectionUtils.isEmpty(deletedPolicies)) { - LOG.warn("Unexpected: found no policy or multiple policies for CHANGE_TYPE_POLICY_UPDATE: " + Arrays.toString(deletedPolicies.toArray())); - } - break; - } - case RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE: { - if (CollectionUtils.isEmpty(deletedPolicies)) { - LOG.warn("Unexpected: found no policy for CHANGE_TYPE_POLICY_DELETE: " + Arrays.toString(deletedPolicies.toArray())); - } - break; - } - default: - break; - } - - if (changeType != RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE) { - ret.add(delta.getPolicy()); - } - } else { - LOG.warn("Found unexpected changeType in policyDelta:[" + delta + "]. Ignoring delta"); - } - } - } else { - if (LOG.isDebugEnabled()) { - LOG.debug("applyDeltas - none of the deltas is for " + serviceType + ")"); - } - ret = policies; + for (RangerPolicyDelta delta : deltas) { + if (!serviceType.equals(delta.getServiceType())) { + continue; } - } else { - if (LOG.isDebugEnabled()) { - LOG.debug("applyDeltas called with empty deltas. Will return policies without change"); + + switch (delta.getChangeType()) { + case RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE: + case RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE: + updatedPolicies.add(delta.getPolicy()); + break; + default: + LOG.warn("Unexpected changeType in policyDelta: [" + delta + "]. Ignoring delta."); } - ret = policies; } - RangerPerfTracer.log(perf); - if (LOG.isDebugEnabled()) { - LOG.debug("<== applyDeltas(serviceType=" + serviceType + "): " + ret); + LOG.debug("<== applyDeltas(serviceType=" + serviceType + "): " + updatedPolicies); } - return ret; + + return updatedPolicies; } public static boolean isValidDeltas(List deltas, String componentServiceType) { diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java index 77717e1e08..3431cfc1f9 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java @@ -53,6 +53,8 @@ public class ServicePolicies implements java.io.Serializable { private TagPolicies tagPolicies; private Map securityZones; private List policyDeltas; + + private Map deleteDeltas; private Map serviceConfig; /** @@ -181,6 +183,14 @@ public String toString() { public void setPolicyDeltas(List policyDeltas) { this.policyDeltas = policyDeltas; } + public Map getDeleteDeltas() { + return deleteDeltas; + } + + public void setDeleteDeltas(Map deleteDeltas) { + this.deleteDeltas = deleteDeltas; + } + @JsonInclude(JsonInclude.Include.NON_NULL) @XmlRootElement @XmlAccessorType(XmlAccessType.FIELD) @@ -393,16 +403,21 @@ public static ServicePolicies applyDelta(final ServicePolicies servicePolicies, List oldResourcePolicies = policyEngine.getResourcePolicies(); List oldTagPolicies = policyEngine.getTagPolicies(); - List newResourcePolicies = RangerPolicyDeltaUtil.applyDeltas(oldResourcePolicies, servicePolicies.getPolicyDeltas(), servicePolicies.getServiceDef().getName()); + List resourcePoliciesAfterDelete = + RangerPolicyDeltaUtil.deletePoliciesByDelta(oldResourcePolicies, servicePolicies.getDeleteDeltas()); + List newResourcePolicies = + RangerPolicyDeltaUtil.applyDeltas(resourcePoliciesAfterDelete, servicePolicies.getPolicyDeltas(), servicePolicies.getServiceDef().getName()); ret.setPolicies(newResourcePolicies); - final List newTagPolicies; + List newTagPolicies; if (servicePolicies.getTagPolicies() != null) { if (LOG.isDebugEnabled()) { LOG.debug("applyingDeltas for tag policies"); } - newTagPolicies = RangerPolicyDeltaUtil.applyDeltas(oldTagPolicies, servicePolicies.getPolicyDeltas(), servicePolicies.getTagPolicies().getServiceDef().getName()); + List tagPoliciesAfterDelete = + RangerPolicyDeltaUtil.deletePoliciesByDelta(oldTagPolicies, servicePolicies.getDeleteDeltas()); + newTagPolicies = RangerPolicyDeltaUtil.applyDeltas(tagPoliciesAfterDelete, servicePolicies.getPolicyDeltas(), servicePolicies.getTagPolicies().getServiceDef().getName()); } else { if (LOG.isDebugEnabled()) { LOG.debug("No need to apply deltas for tag policies"); @@ -411,45 +426,13 @@ public static ServicePolicies applyDelta(final ServicePolicies servicePolicies, } if (LOG.isDebugEnabled()) { - LOG.debug("New tag policies:[" + Arrays.toString(newTagPolicies.toArray()) + "]"); + LOG.debug("New tag policies: " + newTagPolicies); } if (ret.getTagPolicies() != null) { ret.getTagPolicies().setPolicies(newTagPolicies); } - if (MapUtils.isNotEmpty(servicePolicies.getSecurityZones())) { - Map newSecurityZones = new HashMap<>(); - - for (Map.Entry entry : servicePolicies.getSecurityZones().entrySet()) { - String zoneName = entry.getKey(); - SecurityZoneInfo zoneInfo = entry.getValue(); - - List zoneResourcePolicies = policyEngine.getResourcePolicies(zoneName); - // There are no separate tag-policy-repositories for each zone - - if (LOG.isDebugEnabled()) { - LOG.debug("Applying deltas for security-zone:[" + zoneName + "]"); - } - - final List newZonePolicies = RangerPolicyDeltaUtil.applyDeltas(zoneResourcePolicies, zoneInfo.getPolicyDeltas(), servicePolicies.getServiceDef().getName()); - - if (LOG.isDebugEnabled()) { - LOG.debug("New resource policies for security-zone:[" + zoneName + "], zoneResourcePolicies:[" + Arrays.toString(newZonePolicies.toArray())+ "]"); - } - - SecurityZoneInfo newZoneInfo = new SecurityZoneInfo(); - - newZoneInfo.setZoneName(zoneName); - newZoneInfo.setResources(zoneInfo.getResources()); - newZoneInfo.setPolicies(newZonePolicies); - - newSecurityZones.put(zoneName, newZoneInfo); - } - - ret.setSecurityZones(newSecurityZones); - } - return ret; } } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index bfab5baa6c..86a956704c 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -66,6 +66,7 @@ import java.util.Map; import java.util.Set; import java.util.stream.Collectors; +import java.util.Iterator; import static org.apache.atlas.repository.Constants.NAME; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; @@ -142,7 +143,7 @@ public CachePolicyTransformerImpl(AtlasTypeRegistry typeRegistry) throws AtlasBa this.auditEventToDeltaChangeType = new HashMap<>(); this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_CREATE, RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE); - this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_UPDATE, RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE); + this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_UPDATE, RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE); this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_DELETE, RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE); this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_PURGE, RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE); } @@ -176,6 +177,8 @@ public ServicePolicies getPoliciesDelta(String serviceName, Map atlasServicePolicies = allAtlasPolicies.stream().filter(x -> serviceName.equals(x.getAttribute(ATTR_POLICY_SERVICE_NAME))).collect(Collectors.toList()); List policiesDelta = getRangerPolicyDelta(service, policyChanges, atlasServicePolicies); + Map deletedPolicyDeltas = getRangerPolicyDeleteDelta(service, policyChanges); + servicePolicies.setDeleteDeltas(deletedPolicyDeltas); // Process tag based policies String tagServiceName = (String) service.getAttribute(ATTR_SERVICE_TAG_SERVICE); @@ -293,6 +296,42 @@ private List getServicePolicies(AtlasEntityHeader service, int bat return servicePolicies; } + private Map getRangerPolicyDeleteDelta(AtlasEntityHeader service, Map policyChanges) { + String serviceName = (String) service.getAttribute("name"); + String serviceType = (String) service.getAttribute("authServiceType"); + Map policyDeltas = new HashMap<>(); + if (policyChanges.isEmpty()) { + return policyDeltas; + } + + Iterator> iterator = policyChanges.entrySet().iterator(); + while (iterator.hasNext()) { + Map.Entry entry = iterator.next(); + String policyGuid = entry.getKey(); + EntityAuditActionV2 policyChange = entry.getValue(); + + if (policyChange == EntityAuditActionV2.ENTITY_UPDATE || policyChange == EntityAuditActionV2.ENTITY_DELETE + || policyChange == EntityAuditActionV2.ENTITY_PURGE) { + RangerPolicy atlasDeletedPolicy = new RangerPolicy(); + atlasDeletedPolicy.setGuid(policyGuid); + atlasDeletedPolicy.setService(serviceName); + atlasDeletedPolicy.setServiceType(serviceType); + + policyDeltas.put(policyGuid, new RangerPolicyDelta(atlasDeletedPolicy.getId(), + RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE, + atlasDeletedPolicy.getVersion(), + atlasDeletedPolicy)); + } + + if (policyChange == EntityAuditActionV2.ENTITY_DELETE || policyChange == EntityAuditActionV2.ENTITY_PURGE) { + iterator.remove(); // Remove for ENTITY_DELETE and ENTITY_PURGE + } + } + + return policyDeltas; + } + + private List getRangerPolicyDelta(AtlasEntityHeader service, Map policyChanges, List atlasPolicies) throws AtlasBaseException, IOException { String serviceName = (String) service.getAttribute("name"); String serviceType = (String) service.getAttribute("authServiceType"); @@ -303,8 +342,6 @@ private List getRangerPolicyDelta(AtlasEntityHeader service, return policyDeltas; } - ArrayList policyGuids = new ArrayList<>(policyChanges.keySet()); - List rangerPolicies = new ArrayList<>(); if (CollectionUtils.isNotEmpty(atlasPolicies)) { rangerPolicies = transformAtlasPoliciesToRangerPolicies(atlasPolicies, serviceType, serviceName); @@ -316,29 +353,6 @@ private List getRangerPolicyDelta(AtlasEntityHeader service, policyDeltas.add(delta); } - // handle delete changes separately as they won't be present in atlas policies - List deletedPolicyDeltas = new ArrayList<>(); - for (String policyGuid : policyGuids) { - int deltaChangeType = auditEventToDeltaChangeType.get(policyChanges.get(policyGuid)); - if (deltaChangeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE) { - RangerPolicy deletedPolicy = new RangerPolicy(); - deletedPolicy.setGuid(policyGuid); - deletedPolicy.setService(serviceName); - deletedPolicy.setServiceType(serviceType); - RangerPolicyDelta deletedPolicyDelta = new RangerPolicyDelta( - deletedPolicy.getId(), - deltaChangeType, - deletedPolicy.getVersion(), - deletedPolicy - ); - deletedPolicyDeltas.add(deletedPolicyDelta); - } - } - - policyDeltas.addAll(deletedPolicyDeltas); - - LOG.info("PolicyDelta: {}: atlas policies found={}, delta created={}, including deleted policies={}", - serviceName, atlasPolicies.size(), policyDeltas.size(), deletedPolicyDeltas.size()); RequestContext.get().endMetricRecord(recorder); return policyDeltas; From c74e6d4bd5e7e3457de0c0701bb08933d022148b Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Tue, 10 Dec 2024 18:18:30 +0530 Subject: [PATCH 28/31] fix: delete working as expected --- .../atlas/plugin/model/RangerPolicyDelta.java | 3 + .../atlas/plugin/util/ServicePolicies.java | 47 +++++++++++++- .../CachePolicyTransformerImpl.java | 65 ++++++++----------- 3 files changed, 74 insertions(+), 41 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java index b5f9b58233..d1976c9193 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java @@ -76,6 +76,9 @@ public RangerPolicyDelta(final Long id, final Integer changeType, final Long pol @JsonIgnore public String getPolicyGuid() { return policy != null ? policy.getGuid() : null; } + @JsonIgnore + public String getPolicyAtlasGuid() { return policy != null ? policy.getAtlasGuid() : null; } + @JsonIgnore public String getZoneName() { return policy != null ? policy.getZoneName() : null; } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java index 3431cfc1f9..64146db782 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java @@ -397,14 +397,25 @@ static public TagPolicies copyHeader(TagPolicies source, String componentService return ret; } + private static Map fetchDeletedDeltaMap(List deltas) { + Map ret = new HashMap<>(); + for (RangerPolicyDelta delta : deltas) { + if (delta.getChangeType() == RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE || delta.getChangeType() == RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE) { + ret.put(delta.getPolicyAtlasGuid(), delta); + } + } + return ret; + } + public static ServicePolicies applyDelta(final ServicePolicies servicePolicies, RangerPolicyEngineImpl policyEngine) { ServicePolicies ret = copyHeader(servicePolicies); List oldResourcePolicies = policyEngine.getResourcePolicies(); List oldTagPolicies = policyEngine.getTagPolicies(); + Map deletedDeltaMap = fetchDeletedDeltaMap(servicePolicies.getPolicyDeltas()); List resourcePoliciesAfterDelete = - RangerPolicyDeltaUtil.deletePoliciesByDelta(oldResourcePolicies, servicePolicies.getDeleteDeltas()); + RangerPolicyDeltaUtil.deletePoliciesByDelta(oldResourcePolicies, deletedDeltaMap); List newResourcePolicies = RangerPolicyDeltaUtil.applyDeltas(resourcePoliciesAfterDelete, servicePolicies.getPolicyDeltas(), servicePolicies.getServiceDef().getName()); @@ -416,7 +427,7 @@ public static ServicePolicies applyDelta(final ServicePolicies servicePolicies, LOG.debug("applyingDeltas for tag policies"); } List tagPoliciesAfterDelete = - RangerPolicyDeltaUtil.deletePoliciesByDelta(oldTagPolicies, servicePolicies.getDeleteDeltas()); + RangerPolicyDeltaUtil.deletePoliciesByDelta(oldTagPolicies, deletedDeltaMap); newTagPolicies = RangerPolicyDeltaUtil.applyDeltas(tagPoliciesAfterDelete, servicePolicies.getPolicyDeltas(), servicePolicies.getTagPolicies().getServiceDef().getName()); } else { if (LOG.isDebugEnabled()) { @@ -433,6 +444,38 @@ public static ServicePolicies applyDelta(final ServicePolicies servicePolicies, ret.getTagPolicies().setPolicies(newTagPolicies); } + if (MapUtils.isNotEmpty(servicePolicies.getSecurityZones())) { + Map newSecurityZones = new HashMap<>(); + + for (Map.Entry entry : servicePolicies.getSecurityZones().entrySet()) { + String zoneName = entry.getKey(); + SecurityZoneInfo zoneInfo = entry.getValue(); + + List zoneResourcePolicies = policyEngine.getResourcePolicies(zoneName); + // There are no separate tag-policy-repositories for each zone + + if (LOG.isDebugEnabled()) { + LOG.debug("Applying deltas for security-zone:[" + zoneName + "]"); + } + + final List newZonePolicies = RangerPolicyDeltaUtil.applyDeltas(zoneResourcePolicies, zoneInfo.getPolicyDeltas(), servicePolicies.getServiceDef().getName()); + + if (LOG.isDebugEnabled()) { + LOG.debug("New resource policies for security-zone:[" + zoneName + "], zoneResourcePolicies:[" + Arrays.toString(newZonePolicies.toArray())+ "]"); + } + + SecurityZoneInfo newZoneInfo = new SecurityZoneInfo(); + + newZoneInfo.setZoneName(zoneName); + newZoneInfo.setResources(zoneInfo.getResources()); + newZoneInfo.setPolicies(newZonePolicies); + + newSecurityZones.put(zoneName, newZoneInfo); + } + + ret.setSecurityZones(newSecurityZones); + } + return ret; } } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 86a956704c..b85569a780 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -143,7 +143,7 @@ public CachePolicyTransformerImpl(AtlasTypeRegistry typeRegistry) throws AtlasBa this.auditEventToDeltaChangeType = new HashMap<>(); this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_CREATE, RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE); - this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_UPDATE, RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE); + this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_UPDATE, RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE); this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_DELETE, RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE); this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_PURGE, RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE); } @@ -177,8 +177,6 @@ public ServicePolicies getPoliciesDelta(String serviceName, Map atlasServicePolicies = allAtlasPolicies.stream().filter(x -> serviceName.equals(x.getAttribute(ATTR_POLICY_SERVICE_NAME))).collect(Collectors.toList()); List policiesDelta = getRangerPolicyDelta(service, policyChanges, atlasServicePolicies); - Map deletedPolicyDeltas = getRangerPolicyDeleteDelta(service, policyChanges); - servicePolicies.setDeleteDeltas(deletedPolicyDeltas); // Process tag based policies String tagServiceName = (String) service.getAttribute(ATTR_SERVICE_TAG_SERVICE); @@ -296,42 +294,6 @@ private List getServicePolicies(AtlasEntityHeader service, int bat return servicePolicies; } - private Map getRangerPolicyDeleteDelta(AtlasEntityHeader service, Map policyChanges) { - String serviceName = (String) service.getAttribute("name"); - String serviceType = (String) service.getAttribute("authServiceType"); - Map policyDeltas = new HashMap<>(); - if (policyChanges.isEmpty()) { - return policyDeltas; - } - - Iterator> iterator = policyChanges.entrySet().iterator(); - while (iterator.hasNext()) { - Map.Entry entry = iterator.next(); - String policyGuid = entry.getKey(); - EntityAuditActionV2 policyChange = entry.getValue(); - - if (policyChange == EntityAuditActionV2.ENTITY_UPDATE || policyChange == EntityAuditActionV2.ENTITY_DELETE - || policyChange == EntityAuditActionV2.ENTITY_PURGE) { - RangerPolicy atlasDeletedPolicy = new RangerPolicy(); - atlasDeletedPolicy.setGuid(policyGuid); - atlasDeletedPolicy.setService(serviceName); - atlasDeletedPolicy.setServiceType(serviceType); - - policyDeltas.put(policyGuid, new RangerPolicyDelta(atlasDeletedPolicy.getId(), - RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE, - atlasDeletedPolicy.getVersion(), - atlasDeletedPolicy)); - } - - if (policyChange == EntityAuditActionV2.ENTITY_DELETE || policyChange == EntityAuditActionV2.ENTITY_PURGE) { - iterator.remove(); // Remove for ENTITY_DELETE and ENTITY_PURGE - } - } - - return policyDeltas; - } - - private List getRangerPolicyDelta(AtlasEntityHeader service, Map policyChanges, List atlasPolicies) throws AtlasBaseException, IOException { String serviceName = (String) service.getAttribute("name"); String serviceType = (String) service.getAttribute("authServiceType"); @@ -342,6 +304,8 @@ private List getRangerPolicyDelta(AtlasEntityHeader service, return policyDeltas; } + ArrayList policyGuids = new ArrayList<>(policyChanges.keySet()); + List rangerPolicies = new ArrayList<>(); if (CollectionUtils.isNotEmpty(atlasPolicies)) { rangerPolicies = transformAtlasPoliciesToRangerPolicies(atlasPolicies, serviceType, serviceName); @@ -353,6 +317,29 @@ private List getRangerPolicyDelta(AtlasEntityHeader service, policyDeltas.add(delta); } + // handle delete changes separately as they won't be present in atlas policies + List deletedPolicyDeltas = new ArrayList<>(); + for (String policyGuid : policyGuids) { + int deltaChangeType = auditEventToDeltaChangeType.get(policyChanges.get(policyGuid)); + if (deltaChangeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE) { + RangerPolicy deletedPolicy = new RangerPolicy(); + deletedPolicy.setGuid(policyGuid); + deletedPolicy.setService(serviceName); + deletedPolicy.setServiceType(serviceType); + RangerPolicyDelta deletedPolicyDelta = new RangerPolicyDelta( + deletedPolicy.getId(), + deltaChangeType, + deletedPolicy.getVersion(), + deletedPolicy + ); + deletedPolicyDeltas.add(deletedPolicyDelta); + } + } + + policyDeltas.addAll(deletedPolicyDeltas); + + LOG.info("PolicyDelta: {}: atlas policies found={}, delta created={}, including deleted policies={}", + serviceName, atlasPolicies.size(), policyDeltas.size(), deletedPolicyDeltas.size()); RequestContext.get().endMetricRecord(recorder); return policyDeltas; From 37d92550f7ea24ad07b893b9a70c60189f670144 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Tue, 10 Dec 2024 18:45:39 +0530 Subject: [PATCH 29/31] fix: create policy engine from scratch in case of updation --- .../org/apache/atlas/plugin/service/RangerBasePlugin.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java index 76ee3c3e1f..4dd41bcd40 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java @@ -376,10 +376,8 @@ public void setPolicies(ServicePolicies policies) { } if (oldPolicyEngine != null) { - RangerPolicyEngineImpl oldPolicyEngineImpl = (RangerPolicyEngineImpl) oldPolicyEngine; - - newPolicyEngine = RangerPolicyEngineImpl.getPolicyEngine(oldPolicyEngineImpl, policies); - //TODO: this looks like a mistake, second arg should be servicePolicies which has the applied delta + // Create new evaluator for the updated policies + newPolicyEngine = new RangerPolicyEngineImpl(servicePolicies, pluginContext, roles); } if (newPolicyEngine != null) { From 2e5048921d56ce5fb81ce33cb6702b8807d2b68d Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Tue, 10 Dec 2024 21:13:13 +0530 Subject: [PATCH 30/31] nit: few refactors and policy saving based name instead of types --- .../plugin/service/RangerBasePlugin.java | 3 +- .../plugin/util/RangerPolicyDeltaUtil.java | 10 ++-- .../atlas/plugin/util/ServicePolicies.java | 53 +++---------------- .../CachePolicyTransformerImpl.java | 10 +--- 4 files changed, 16 insertions(+), 60 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java index 4dd41bcd40..af0aa716e1 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java @@ -428,7 +428,8 @@ public void setPolicies(ServicePolicies policies) { } } catch (Exception e) { - LOG.error("setPolicies: policy engine initialization failed! Leaving current policy engine as-is. Exception : ", e); + LOG.error("setPolicies: Failed to set policies, didn't set policies", e); + throw e; } if (LOG.isDebugEnabled()) { LOG.debug("<== setPolicies(" + policies + ")"); diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java index a37187aee8..d1e5ebf03a 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java @@ -49,7 +49,7 @@ public static List deletePoliciesByDelta(List polici } } - public static List applyDeltas(List policies, List deltas, String serviceType) { + public static List applyDeltas(List policies, List deltas, String serviceType, String serviceName) { if (LOG.isDebugEnabled()) { LOG.debug("==> applyDeltas(serviceType=" + serviceType + ")"); } @@ -61,9 +61,9 @@ public static List applyDeltas(List policies, List serviceType.equals(delta.getServiceType())); + boolean hasExpectedServiceName = deltas.stream().anyMatch(delta -> serviceName.equals(delta.getPolicy().getService())); - if (!hasExpectedServiceType) { + if (!hasExpectedServiceName) { if (LOG.isDebugEnabled()) { LOG.debug("No deltas match the expected serviceType: " + serviceType); } @@ -73,7 +73,7 @@ public static List applyDeltas(List policies, List updatedPolicies = new ArrayList<>(policies); for (RangerPolicyDelta delta : deltas) { - if (!serviceType.equals(delta.getServiceType())) { + if (!serviceName.equals(delta.getPolicy().getService())) { continue; } @@ -83,7 +83,7 @@ public static List applyDeltas(List policies, List securityZones; private List policyDeltas; - - private Map deleteDeltas; private Map serviceConfig; /** @@ -183,14 +186,6 @@ public String toString() { public void setPolicyDeltas(List policyDeltas) { this.policyDeltas = policyDeltas; } - public Map getDeleteDeltas() { - return deleteDeltas; - } - - public void setDeleteDeltas(Map deleteDeltas) { - this.deleteDeltas = deleteDeltas; - } - @JsonInclude(JsonInclude.Include.NON_NULL) @XmlRootElement @XmlAccessorType(XmlAccessType.FIELD) @@ -417,7 +412,7 @@ public static ServicePolicies applyDelta(final ServicePolicies servicePolicies, List resourcePoliciesAfterDelete = RangerPolicyDeltaUtil.deletePoliciesByDelta(oldResourcePolicies, deletedDeltaMap); List newResourcePolicies = - RangerPolicyDeltaUtil.applyDeltas(resourcePoliciesAfterDelete, servicePolicies.getPolicyDeltas(), servicePolicies.getServiceDef().getName()); + RangerPolicyDeltaUtil.applyDeltas(resourcePoliciesAfterDelete, servicePolicies.getPolicyDeltas(), servicePolicies.getServiceDef().getName(), servicePolicies.getServiceName()); ret.setPolicies(newResourcePolicies); @@ -428,7 +423,7 @@ public static ServicePolicies applyDelta(final ServicePolicies servicePolicies, } List tagPoliciesAfterDelete = RangerPolicyDeltaUtil.deletePoliciesByDelta(oldTagPolicies, deletedDeltaMap); - newTagPolicies = RangerPolicyDeltaUtil.applyDeltas(tagPoliciesAfterDelete, servicePolicies.getPolicyDeltas(), servicePolicies.getTagPolicies().getServiceDef().getName()); + newTagPolicies = RangerPolicyDeltaUtil.applyDeltas(tagPoliciesAfterDelete, servicePolicies.getPolicyDeltas(), servicePolicies.getTagPolicies().getServiceDef().getName(), servicePolicies.getTagPolicies().getServiceName()); } else { if (LOG.isDebugEnabled()) { LOG.debug("No need to apply deltas for tag policies"); @@ -444,38 +439,6 @@ public static ServicePolicies applyDelta(final ServicePolicies servicePolicies, ret.getTagPolicies().setPolicies(newTagPolicies); } - if (MapUtils.isNotEmpty(servicePolicies.getSecurityZones())) { - Map newSecurityZones = new HashMap<>(); - - for (Map.Entry entry : servicePolicies.getSecurityZones().entrySet()) { - String zoneName = entry.getKey(); - SecurityZoneInfo zoneInfo = entry.getValue(); - - List zoneResourcePolicies = policyEngine.getResourcePolicies(zoneName); - // There are no separate tag-policy-repositories for each zone - - if (LOG.isDebugEnabled()) { - LOG.debug("Applying deltas for security-zone:[" + zoneName + "]"); - } - - final List newZonePolicies = RangerPolicyDeltaUtil.applyDeltas(zoneResourcePolicies, zoneInfo.getPolicyDeltas(), servicePolicies.getServiceDef().getName()); - - if (LOG.isDebugEnabled()) { - LOG.debug("New resource policies for security-zone:[" + zoneName + "], zoneResourcePolicies:[" + Arrays.toString(newZonePolicies.toArray())+ "]"); - } - - SecurityZoneInfo newZoneInfo = new SecurityZoneInfo(); - - newZoneInfo.setZoneName(zoneName); - newZoneInfo.setResources(zoneInfo.getResources()); - newZoneInfo.setPolicies(newZonePolicies); - - newSecurityZones.put(zoneName, newZoneInfo); - } - - ret.setSecurityZones(newSecurityZones); - } - return ret; } } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index b85569a780..71e798091b 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -708,17 +708,9 @@ private Map getMap(String key, Object value) { private RangerPolicy getRangerPolicy(AtlasEntityHeader atlasPolicy, String serviceType) { RangerPolicy policy = new RangerPolicy(); - //policy.setId(atlasPolicy.getGuid()); policy.setName((String) atlasPolicy.getAttribute(QUALIFIED_NAME)); policy.setService((String) atlasPolicy.getAttribute(ATTR_POLICY_SERVICE_NAME)); - - // Adding atlas as serviceType for tag policies, as atlas_tag doesn't have all the resource available for evaluation - if (serviceType != null && serviceType.equals(TAG_RESOURCE_NAME) && policy.getService().equals("atlas")) { - policy.setServiceType("atlas"); - } else { - policy.setServiceType(serviceType); - } - + policy.setServiceType(serviceType); policy.setGuid(atlasPolicy.getGuid()); policy.setCreatedBy(atlasPolicy.getCreatedBy()); policy.setCreateTime(atlasPolicy.getCreateTime()); From 0577ea18462f7f55c65158bfd83249c6ac97334d Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Tue, 10 Dec 2024 21:13:48 +0530 Subject: [PATCH 31/31] nit: add exceptions in setpolicies incase of failure it will retry --- .../atlas/plugin/util/PolicyRefresher.java | 21 +++++++++---------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java index ed5b1a3687..99deb21d2d 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -37,7 +37,6 @@ import java.io.FileWriter; import java.io.Reader; import java.io.Writer; -import java.util.Date; import java.util.Timer; import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; @@ -62,7 +61,7 @@ public class PolicyRefresher extends Thread { private final BlockingQueue policyDownloadQueue = new LinkedBlockingQueue<>(); private Timer policyDownloadTimer; private long lastKnownVersion = -1L; - private long lastUpdatedTiemInMillis = -1L; + private long lastUpdatedTimeInMillis = -1L; private long lastActivationTimeInMillis; private boolean policiesSetInPlugin; private boolean serviceDefSetInPlugin; @@ -222,7 +221,7 @@ public void run() { loadPolicy(); loadUserStore(); } catch(InterruptedException excp) { - LOG.info("PolicyRefresher(serviceName=" + serviceName + ").run(): interrupted! Exiting thread", excp); + LOG.info("PolicyRefreshxer(serviceName=" + serviceName + ").run(): interrupted! Exiting thread", excp); break; } finally { if (trigger != null) { @@ -279,7 +278,7 @@ private void loadPolicy() { serviceDefSetInPlugin = false; setLastActivationTimeInMillis(System.currentTimeMillis()); lastKnownVersion = svcPolicies.getPolicyVersion() != null ? svcPolicies.getPolicyVersion() : -1L; - lastUpdatedTiemInMillis = svcPolicies.getPolicyUpdateTime() != null ? svcPolicies.getPolicyUpdateTime().getTime() : -1L; + lastUpdatedTimeInMillis = svcPolicies.getPolicyUpdateTime() != null ? svcPolicies.getPolicyUpdateTime().getTime() : -1L; } else { if (!policiesSetInPlugin && !serviceDefSetInPlugin) { plugIn.setPolicies(null); @@ -293,10 +292,10 @@ private void loadPolicy() { serviceDefSetInPlugin = true; setLastActivationTimeInMillis(System.currentTimeMillis()); lastKnownVersion = -1; - lastUpdatedTiemInMillis = -1; + lastUpdatedTimeInMillis = -1; } } catch (Exception excp) { - LOG.error("Encountered unexpected exception, ignoring..", excp); + LOG.error("Encountered unexpected exception!!!!!!!!!!!", excp); } RangerPerfTracer.log(perf); @@ -323,17 +322,17 @@ private ServicePolicies loadPolicyfromPolicyAdmin() throws RangerServiceNotFound try { - if (serviceName.equals("atlas") && plugIn.getTypeRegistry() != null && lastUpdatedTiemInMillis == -1) { + if (serviceName.equals("atlas") && plugIn.getTypeRegistry() != null && lastUpdatedTimeInMillis == -1) { LOG.info("PolicyRefresher(serviceName=" + serviceName + "): loading all policies for first time"); RangerRESTUtils restUtils = new RangerRESTUtils(); CachePolicyTransformerImpl transformer = new CachePolicyTransformerImpl(plugIn.getTypeRegistry()); svcPolicies = transformer.getPoliciesAll(serviceName, restUtils.getPluginId(serviceName, plugIn.getAppId()), - lastUpdatedTiemInMillis); + lastUpdatedTimeInMillis); } else { - LOG.info("PolicyRefresher(serviceName=" + serviceName + "): loading delta policies from last known version=" + lastKnownVersion + ", lastUpdatedTime=" + lastUpdatedTiemInMillis); - svcPolicies = atlasAuthAdminClient.getServicePoliciesIfUpdated(lastUpdatedTiemInMillis, this.enableDeltaBasedRefresh); + LOG.info("PolicyRefresher(serviceName=" + serviceName + "): loading delta policies from last known version=" + lastKnownVersion + ", lastUpdatedTime=" + lastUpdatedTimeInMillis); + svcPolicies = atlasAuthAdminClient.getServicePoliciesIfUpdated(lastUpdatedTimeInMillis, this.enableDeltaBasedRefresh); } boolean isUpdated = svcPolicies != null; @@ -400,7 +399,7 @@ private ServicePolicies loadFromCache() { } lastKnownVersion = policies.getPolicyVersion() == null ? -1 : policies.getPolicyVersion().longValue(); - lastUpdatedTiemInMillis = policies.getPolicyUpdateTime() == null ? -1 : policies.getPolicyUpdateTime().getTime(); + lastUpdatedTimeInMillis = policies.getPolicyUpdateTime() == null ? -1 : policies.getPolicyUpdateTime().getTime(); } } catch (Exception excp) { LOG.error("failed to load policies from cache file " + cacheFile.getAbsolutePath(), excp);