Skip to content

Commit

Permalink
feat: update policy delta logic full revamp
Browse files Browse the repository at this point in the history
  • Loading branch information
@sumandas0
sumandas0 committed Dec 10, 2024
1 parent ed4a51e commit 8df0b5e
Show file tree
Hide file tree
Showing 5 changed files with 96 additions and 160 deletions.
5 changes: 1 addition & 4 deletions auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -532,10 +532,7 @@ public void setIsDenyAllElse(Boolean isDenyAllElse) {

@JsonIgnore
public String getAtlasGuid() {
if (getGuid().length() > 36) {
return getGuid().substring(0, 36);
}
return getGuid();
return getGuid().length() > 36 ? getGuid().substring(0, 36) : getGuid();
}

@Override
Expand Down
1 change: 0 additions & 1 deletion ...nts-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyRepository.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1355,7 +1355,6 @@ private void deletePolicyEvaluator(RangerPolicyEvaluator evaluator) {
}

private RangerPolicyEvaluator update(final RangerPolicyDelta delta, final RangerPolicyEvaluator currentEvaluator) {
LOG.info("PolicyDelta: RangerPolicyRepository.update is called, policyGuid: "+delta.getPolicyGuid());
if (LOG.isDebugEnabled()) {
LOG.debug("==> RangerPolicyRepository.update(delta=" + delta + ", currentEvaluator=" + (currentEvaluator == null ? null : currentEvaluator.getPolicy()) + ")");
}
Expand Down
129 changes: 36 additions & 93 deletions auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,125 +30,68 @@

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Iterator;
import java.util.List;
import java.util.Map;

public class RangerPolicyDeltaUtil {

private static final Log LOG = LogFactory.getLog(RangerPolicyDeltaUtil.class);

private static final Log PERF_POLICY_DELTA_LOG = RangerPerfTracer.getPerfLogger("policy.delta");

public static List<RangerPolicy> deletePoliciesByDelta(List<RangerPolicy> policies, Map<String, RangerPolicyDelta> deltas) {
if (MapUtils.isNotEmpty(deltas)) {
List<RangerPolicy> ret = new ArrayList<>(policies);
ret.removeIf(policy -> deltas.containsKey(policy.getAtlasGuid()));
return ret;
} else {
return policies;
}
}

public static List<RangerPolicy> applyDeltas(List<RangerPolicy> policies, List<RangerPolicyDelta> deltas, String serviceType) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> applyDeltas(serviceType=" + serviceType + ")");
}

List<RangerPolicy> ret;

RangerPerfTracer perf = null;

if(RangerPerfTracer.isPerfTraceEnabled(PERF_POLICY_DELTA_LOG)) {
perf = RangerPerfTracer.getPerfTracer(PERF_POLICY_DELTA_LOG, "RangerPolicyDelta.applyDeltas()");
if (CollectionUtils.isEmpty(deltas)) {
if (LOG.isDebugEnabled()) {
LOG.debug("applyDeltas called with empty deltas. Returning policies without changes.");
}
return policies;
}

boolean hasExpectedServiceType = false;
boolean hasExpectedServiceType = deltas.stream().anyMatch(delta -> serviceType.equals(delta.getServiceType()));

if (CollectionUtils.isNotEmpty(deltas)) {
if (!hasExpectedServiceType) {
if (LOG.isDebugEnabled()) {
LOG.debug("applyDeltas(deltas=" + Arrays.toString(deltas.toArray()) + ", serviceType=" + serviceType + ")");
}

for (RangerPolicyDelta delta : deltas) {
if (serviceType.equals(delta.getServiceType())) {
hasExpectedServiceType = true;
break;
} else if (!serviceType.equals(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME) && !delta.getServiceType().equals(EmbeddedServiceDefsUtil.EMBEDDED_SERVICEDEF_TAG_NAME)) {
LOG.warn("Found unexpected serviceType in policyDelta:[" + delta + "]. Was expecting serviceType:[" + serviceType + "]. Should NOT have come here!! Ignoring delta and continuing");
}
LOG.debug("No deltas match the expected serviceType: " + serviceType);
}
return policies;
}

if (hasExpectedServiceType) {
ret = new ArrayList<>(policies);
List<RangerPolicy> updatedPolicies = new ArrayList<>(policies);

for (RangerPolicyDelta delta : deltas) {
if (!serviceType.equals(delta.getServiceType())) {
continue;
}

int changeType = delta.getChangeType();

if (changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE || changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE || changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE) {
String policyId = delta.getPolicyGuid(); // change to getGuid() as id is not set in policy

if (policyId == null) {
continue;
}

List<RangerPolicy> deletedPolicies = new ArrayList<>();

Iterator<RangerPolicy> iter = ret.iterator();

while (iter.hasNext()) {
RangerPolicy policy = iter.next();
if (
(changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE && policyId.equals(policy.getAtlasGuid())) ||
(changeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE && policyId.equals(policy.getAtlasGuid()))
){
deletedPolicies.add(policy);
iter.remove();
}
}

switch(changeType) {
case RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE: {
if (CollectionUtils.isNotEmpty(deletedPolicies)) {
LOG.warn("Unexpected: found existing policy for CHANGE_TYPE_POLICY_CREATE: " + Arrays.toString(deletedPolicies.toArray()));
}
break;
}
case RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE: {
if (CollectionUtils.isEmpty(deletedPolicies)) {
LOG.warn("Unexpected: found no policy or multiple policies for CHANGE_TYPE_POLICY_UPDATE: " + Arrays.toString(deletedPolicies.toArray()));
}
break;
}
case RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE: {
if (CollectionUtils.isEmpty(deletedPolicies)) {
LOG.warn("Unexpected: found no policy for CHANGE_TYPE_POLICY_DELETE: " + Arrays.toString(deletedPolicies.toArray()));
}
break;
}
default:
break;
}

if (changeType != RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE) {
ret.add(delta.getPolicy());
}
} else {
LOG.warn("Found unexpected changeType in policyDelta:[" + delta + "]. Ignoring delta");
}
}
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("applyDeltas - none of the deltas is for " + serviceType + ")");
}
ret = policies;
for (RangerPolicyDelta delta : deltas) {
if (!serviceType.equals(delta.getServiceType())) {
continue;
}
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("applyDeltas called with empty deltas. Will return policies without change");

switch (delta.getChangeType()) {
case RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE:
case RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE:
updatedPolicies.add(delta.getPolicy());
break;
default:
LOG.warn("Unexpected changeType in policyDelta: [" + delta + "]. Ignoring delta.");
}
ret = policies;
}

RangerPerfTracer.log(perf);

if (LOG.isDebugEnabled()) {
LOG.debug("<== applyDeltas(serviceType=" + serviceType + "): " + ret);
LOG.debug("<== applyDeltas(serviceType=" + serviceType + "): " + updatedPolicies);
}
return ret;

return updatedPolicies;
}

public static boolean isValidDeltas(List<RangerPolicyDelta> deltas, String componentServiceType) {
Expand Down
55 changes: 19 additions & 36 deletions auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,8 @@ public class ServicePolicies implements java.io.Serializable {
private TagPolicies tagPolicies;
private Map<String, SecurityZoneInfo> securityZones;
private List<RangerPolicyDelta> policyDeltas;

private Map<String, RangerPolicyDelta> deleteDeltas;
private Map<String, String> serviceConfig;

/**
Expand Down Expand Up @@ -181,6 +183,14 @@ public String toString() {

public void setPolicyDeltas(List<RangerPolicyDelta> policyDeltas) { this.policyDeltas = policyDeltas; }

public Map<String, RangerPolicyDelta> getDeleteDeltas() {
return deleteDeltas;
}

public void setDeleteDeltas(Map<String, RangerPolicyDelta> deleteDeltas) {
this.deleteDeltas = deleteDeltas;
}

@JsonInclude(JsonInclude.Include.NON_NULL)
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
Expand Down Expand Up @@ -393,16 +403,21 @@ public static ServicePolicies applyDelta(final ServicePolicies servicePolicies,
List<RangerPolicy> oldResourcePolicies = policyEngine.getResourcePolicies();
List<RangerPolicy> oldTagPolicies = policyEngine.getTagPolicies();

List<RangerPolicy> newResourcePolicies = RangerPolicyDeltaUtil.applyDeltas(oldResourcePolicies, servicePolicies.getPolicyDeltas(), servicePolicies.getServiceDef().getName());
List<RangerPolicy> resourcePoliciesAfterDelete =
RangerPolicyDeltaUtil.deletePoliciesByDelta(oldResourcePolicies, servicePolicies.getDeleteDeltas());
List<RangerPolicy> newResourcePolicies =
RangerPolicyDeltaUtil.applyDeltas(resourcePoliciesAfterDelete, servicePolicies.getPolicyDeltas(), servicePolicies.getServiceDef().getName());

ret.setPolicies(newResourcePolicies);

final List<RangerPolicy> newTagPolicies;
List<RangerPolicy> newTagPolicies;
if (servicePolicies.getTagPolicies() != null) {
if (LOG.isDebugEnabled()) {
LOG.debug("applyingDeltas for tag policies");
}
newTagPolicies = RangerPolicyDeltaUtil.applyDeltas(oldTagPolicies, servicePolicies.getPolicyDeltas(), servicePolicies.getTagPolicies().getServiceDef().getName());
List<RangerPolicy> tagPoliciesAfterDelete =
RangerPolicyDeltaUtil.deletePoliciesByDelta(oldTagPolicies, servicePolicies.getDeleteDeltas());
newTagPolicies = RangerPolicyDeltaUtil.applyDeltas(tagPoliciesAfterDelete, servicePolicies.getPolicyDeltas(), servicePolicies.getTagPolicies().getServiceDef().getName());
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("No need to apply deltas for tag policies");
Expand All @@ -411,45 +426,13 @@ public static ServicePolicies applyDelta(final ServicePolicies servicePolicies,
}

if (LOG.isDebugEnabled()) {
LOG.debug("New tag policies:[" + Arrays.toString(newTagPolicies.toArray()) + "]");
LOG.debug("New tag policies: " + newTagPolicies);
}

if (ret.getTagPolicies() != null) {
ret.getTagPolicies().setPolicies(newTagPolicies);
}

if (MapUtils.isNotEmpty(servicePolicies.getSecurityZones())) {
Map<String, SecurityZoneInfo> newSecurityZones = new HashMap<>();

for (Map.Entry<String, SecurityZoneInfo> entry : servicePolicies.getSecurityZones().entrySet()) {
String zoneName = entry.getKey();
SecurityZoneInfo zoneInfo = entry.getValue();

List<RangerPolicy> zoneResourcePolicies = policyEngine.getResourcePolicies(zoneName);
// There are no separate tag-policy-repositories for each zone

if (LOG.isDebugEnabled()) {
LOG.debug("Applying deltas for security-zone:[" + zoneName + "]");
}

final List<RangerPolicy> newZonePolicies = RangerPolicyDeltaUtil.applyDeltas(zoneResourcePolicies, zoneInfo.getPolicyDeltas(), servicePolicies.getServiceDef().getName());

if (LOG.isDebugEnabled()) {
LOG.debug("New resource policies for security-zone:[" + zoneName + "], zoneResourcePolicies:[" + Arrays.toString(newZonePolicies.toArray())+ "]");
}

SecurityZoneInfo newZoneInfo = new SecurityZoneInfo();

newZoneInfo.setZoneName(zoneName);
newZoneInfo.setResources(zoneInfo.getResources());
newZoneInfo.setPolicies(newZonePolicies);

newSecurityZones.put(zoneName, newZoneInfo);
}

ret.setSecurityZones(newSecurityZones);
}

return ret;
}
}
66 changes: 40 additions & 26 deletions ...s-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -66,6 +66,7 @@
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.Iterator;

import static org.apache.atlas.repository.Constants.NAME;
import static org.apache.atlas.repository.Constants.QUALIFIED_NAME;
Expand Down Expand Up @@ -142,7 +143,7 @@ public CachePolicyTransformerImpl(AtlasTypeRegistry typeRegistry) throws AtlasBa

this.auditEventToDeltaChangeType = new HashMap<>();
this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_CREATE, RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE);
this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_UPDATE, RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE);
this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_UPDATE, RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE);
this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_DELETE, RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE);
this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_PURGE, RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE);
}
Expand Down Expand Up @@ -176,6 +177,8 @@ public ServicePolicies getPoliciesDelta(String serviceName, Map<String, EntityAu

List<AtlasEntityHeader> atlasServicePolicies = allAtlasPolicies.stream().filter(x -> serviceName.equals(x.getAttribute(ATTR_POLICY_SERVICE_NAME))).collect(Collectors.toList());
List<RangerPolicyDelta> policiesDelta = getRangerPolicyDelta(service, policyChanges, atlasServicePolicies);
Map<String, RangerPolicyDelta> deletedPolicyDeltas = getRangerPolicyDeleteDelta(service, policyChanges);
servicePolicies.setDeleteDeltas(deletedPolicyDeltas);

// Process tag based policies
String tagServiceName = (String) service.getAttribute(ATTR_SERVICE_TAG_SERVICE);
Expand Down Expand Up @@ -293,6 +296,42 @@ private List<RangerPolicy> getServicePolicies(AtlasEntityHeader service, int bat
return servicePolicies;
}

private Map<String, RangerPolicyDelta> getRangerPolicyDeleteDelta(AtlasEntityHeader service, Map<String, EntityAuditActionV2> policyChanges) {
String serviceName = (String) service.getAttribute("name");
String serviceType = (String) service.getAttribute("authServiceType");
Map<String, RangerPolicyDelta> policyDeltas = new HashMap<>();
if (policyChanges.isEmpty()) {
return policyDeltas;
}

Iterator<Map.Entry<String, EntityAuditActionV2>> iterator = policyChanges.entrySet().iterator();
while (iterator.hasNext()) {
Map.Entry<String, EntityAuditActionV2> entry = iterator.next();
String policyGuid = entry.getKey();
EntityAuditActionV2 policyChange = entry.getValue();

if (policyChange == EntityAuditActionV2.ENTITY_UPDATE || policyChange == EntityAuditActionV2.ENTITY_DELETE
|| policyChange == EntityAuditActionV2.ENTITY_PURGE) {
RangerPolicy atlasDeletedPolicy = new RangerPolicy();
atlasDeletedPolicy.setGuid(policyGuid);
atlasDeletedPolicy.setService(serviceName);
atlasDeletedPolicy.setServiceType(serviceType);

policyDeltas.put(policyGuid, new RangerPolicyDelta(atlasDeletedPolicy.getId(),
RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE,
atlasDeletedPolicy.getVersion(),
atlasDeletedPolicy));
}

if (policyChange == EntityAuditActionV2.ENTITY_DELETE || policyChange == EntityAuditActionV2.ENTITY_PURGE) {
iterator.remove(); // Remove for ENTITY_DELETE and ENTITY_PURGE
}
}

return policyDeltas;
}


private List<RangerPolicyDelta> getRangerPolicyDelta(AtlasEntityHeader service, Map<String, EntityAuditActionV2> policyChanges, List<AtlasEntityHeader> atlasPolicies) throws AtlasBaseException, IOException {
String serviceName = (String) service.getAttribute("name");
String serviceType = (String) service.getAttribute("authServiceType");
Expand All @@ -303,8 +342,6 @@ private List<RangerPolicyDelta> getRangerPolicyDelta(AtlasEntityHeader service,
return policyDeltas;
}

ArrayList<String> policyGuids = new ArrayList<>(policyChanges.keySet());

List<RangerPolicy> rangerPolicies = new ArrayList<>();
if (CollectionUtils.isNotEmpty(atlasPolicies)) {
rangerPolicies = transformAtlasPoliciesToRangerPolicies(atlasPolicies, serviceType, serviceName);
Expand All @@ -316,29 +353,6 @@ private List<RangerPolicyDelta> getRangerPolicyDelta(AtlasEntityHeader service,
policyDeltas.add(delta);
}

// handle delete changes separately as they won't be present in atlas policies
List<RangerPolicyDelta> deletedPolicyDeltas = new ArrayList<>();
for (String policyGuid : policyGuids) {
int deltaChangeType = auditEventToDeltaChangeType.get(policyChanges.get(policyGuid));
if (deltaChangeType == RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE) {
RangerPolicy deletedPolicy = new RangerPolicy();
deletedPolicy.setGuid(policyGuid);
deletedPolicy.setService(serviceName);
deletedPolicy.setServiceType(serviceType);
RangerPolicyDelta deletedPolicyDelta = new RangerPolicyDelta(
deletedPolicy.getId(),
deltaChangeType,
deletedPolicy.getVersion(),
deletedPolicy
);
deletedPolicyDeltas.add(deletedPolicyDelta);
}
}

policyDeltas.addAll(deletedPolicyDeltas);

LOG.info("PolicyDelta: {}: atlas policies found={}, delta created={}, including deleted policies={}",
serviceName, atlasPolicies.size(), policyDeltas.size(), deletedPolicyDeltas.size());
RequestContext.get().endMetricRecord(recorder);

return policyDeltas;
Expand Down

0 comments on commit 8df0b5e

Please sign in to comment.