Skip to content

Commit

Permalink
Merge pull request #3872 from atlanhq/ns/feat/delta-policy-master
Browse files Browse the repository at this point in the history 
DG-1709 | Set tag policies and policies based on service name instead of serviceType
  • Loading branch information
@krsoninikhil
krsoninikhil authored Dec 10, 2024
2 parents e708088 + 0577ea1 commit e44b399
Show file tree
Hide file tree
Showing 5 changed files with 26 additions and 71 deletions.
3 changes: 2 additions & 1 deletion auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -428,7 +428,8 @@ public void setPolicies(ServicePolicies policies) {
}

} catch (Exception e) {
LOG.error("setPolicies: policy engine initialization failed! Leaving current policy engine as-is. Exception : ", e);
LOG.error("setPolicies: Failed to set policies, didn't set policies", e);
throw e;
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== setPolicies(" + policies + ")");
Expand Down
21 changes: 10 additions & 11 deletions auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@
import java.io.FileWriter;
import java.io.Reader;
import java.io.Writer;
import java.util.Date;
import java.util.Timer;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
Expand All @@ -62,7 +61,7 @@ public class PolicyRefresher extends Thread {
private final BlockingQueue<DownloadTrigger> policyDownloadQueue = new LinkedBlockingQueue<>();
private Timer policyDownloadTimer;
private long lastKnownVersion = -1L;
private long lastUpdatedTiemInMillis = -1L;
private long lastUpdatedTimeInMillis = -1L;
private long lastActivationTimeInMillis;
private boolean policiesSetInPlugin;
private boolean serviceDefSetInPlugin;
Expand Down Expand Up @@ -222,7 +221,7 @@ public void run() {
loadPolicy();
loadUserStore();
} catch(InterruptedException excp) {
LOG.info("PolicyRefresher(serviceName=" + serviceName + ").run(): interrupted! Exiting thread", excp);
LOG.info("PolicyRefreshxer(serviceName=" + serviceName + ").run(): interrupted! Exiting thread", excp);
break;
} finally {
if (trigger != null) {
Expand Down Expand Up @@ -279,7 +278,7 @@ private void loadPolicy() {
serviceDefSetInPlugin = false;
setLastActivationTimeInMillis(System.currentTimeMillis());
lastKnownVersion = svcPolicies.getPolicyVersion() != null ? svcPolicies.getPolicyVersion() : -1L;
lastUpdatedTiemInMillis = svcPolicies.getPolicyUpdateTime() != null ? svcPolicies.getPolicyUpdateTime().getTime() : -1L;
lastUpdatedTimeInMillis = svcPolicies.getPolicyUpdateTime() != null ? svcPolicies.getPolicyUpdateTime().getTime() : -1L;
} else {
if (!policiesSetInPlugin && !serviceDefSetInPlugin) {
plugIn.setPolicies(null);
Expand All @@ -293,10 +292,10 @@ private void loadPolicy() {
serviceDefSetInPlugin = true;
setLastActivationTimeInMillis(System.currentTimeMillis());
lastKnownVersion = -1;
lastUpdatedTiemInMillis = -1;
lastUpdatedTimeInMillis = -1;
}
} catch (Exception excp) {
LOG.error("Encountered unexpected exception, ignoring..", excp);
LOG.error("Encountered unexpected exception!!!!!!!!!!!", excp);
}

RangerPerfTracer.log(perf);
Expand All @@ -323,17 +322,17 @@ private ServicePolicies loadPolicyfromPolicyAdmin() throws RangerServiceNotFound
try {


if (serviceName.equals("atlas") && plugIn.getTypeRegistry() != null && lastUpdatedTiemInMillis == -1) {
if (serviceName.equals("atlas") && plugIn.getTypeRegistry() != null && lastUpdatedTimeInMillis == -1) {
LOG.info("PolicyRefresher(serviceName=" + serviceName + "): loading all policies for first time");
RangerRESTUtils restUtils = new RangerRESTUtils();
CachePolicyTransformerImpl transformer = new CachePolicyTransformerImpl(plugIn.getTypeRegistry());

svcPolicies = transformer.getPoliciesAll(serviceName,
restUtils.getPluginId(serviceName, plugIn.getAppId()),
lastUpdatedTiemInMillis);
lastUpdatedTimeInMillis);
} else {
LOG.info("PolicyRefresher(serviceName=" + serviceName + "): loading delta policies from last known version=" + lastKnownVersion + ", lastUpdatedTime=" + lastUpdatedTiemInMillis);
svcPolicies = atlasAuthAdminClient.getServicePoliciesIfUpdated(lastUpdatedTiemInMillis, this.enableDeltaBasedRefresh);
LOG.info("PolicyRefresher(serviceName=" + serviceName + "): loading delta policies from last known version=" + lastKnownVersion + ", lastUpdatedTime=" + lastUpdatedTimeInMillis);
svcPolicies = atlasAuthAdminClient.getServicePoliciesIfUpdated(lastUpdatedTimeInMillis, this.enableDeltaBasedRefresh);
}

boolean isUpdated = svcPolicies != null;
Expand Down Expand Up @@ -400,7 +399,7 @@ private ServicePolicies loadFromCache() {
}

lastKnownVersion = policies.getPolicyVersion() == null ? -1 : policies.getPolicyVersion().longValue();
lastUpdatedTiemInMillis = policies.getPolicyUpdateTime() == null ? -1 : policies.getPolicyUpdateTime().getTime();
lastUpdatedTimeInMillis = policies.getPolicyUpdateTime() == null ? -1 : policies.getPolicyUpdateTime().getTime();
}
} catch (Exception excp) {
LOG.error("failed to load policies from cache file " + cacheFile.getAbsolutePath(), excp);
Expand Down
10 changes: 5 additions & 5 deletions auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ public static List<RangerPolicy> deletePoliciesByDelta(List<RangerPolicy> polici
}
}

public static List<RangerPolicy> applyDeltas(List<RangerPolicy> policies, List<RangerPolicyDelta> deltas, String serviceType) {
public static List<RangerPolicy> applyDeltas(List<RangerPolicy> policies, List<RangerPolicyDelta> deltas, String serviceType, String serviceName) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> applyDeltas(serviceType=" + serviceType + ")");
}
Expand All @@ -61,9 +61,9 @@ public static List<RangerPolicy> applyDeltas(List<RangerPolicy> policies, List<R
return policies;
}

boolean hasExpectedServiceType = deltas.stream().anyMatch(delta -> serviceType.equals(delta.getServiceType()));
boolean hasExpectedServiceName = deltas.stream().anyMatch(delta -> serviceName.equals(delta.getPolicy().getService()));

if (!hasExpectedServiceType) {
if (!hasExpectedServiceName) {
if (LOG.isDebugEnabled()) {
LOG.debug("No deltas match the expected serviceType: " + serviceType);
}
Expand All @@ -73,7 +73,7 @@ public static List<RangerPolicy> applyDeltas(List<RangerPolicy> policies, List<R
List<RangerPolicy> updatedPolicies = new ArrayList<>(policies);

for (RangerPolicyDelta delta : deltas) {
if (!serviceType.equals(delta.getServiceType())) {
if (!serviceName.equals(delta.getPolicy().getService())) {
continue;
}

Expand All @@ -83,7 +83,7 @@ public static List<RangerPolicy> applyDeltas(List<RangerPolicy> policies, List<R
updatedPolicies.add(delta.getPolicy());
break;
default:
LOG.warn("Unexpected changeType in policyDelta: [" + delta + "]. Ignoring delta.");
LOG.warn("Unexpected changeType in policyDelta: [" + delta.getPolicyGuid() + "]. Ignoring delta.");
}
}

Expand Down
53 changes: 8 additions & 45 deletions auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,12 @@
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import java.util.*;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

@JsonInclude(JsonInclude.Include.NON_NULL)
@XmlRootElement
Expand All @@ -53,8 +58,6 @@ public class ServicePolicies implements java.io.Serializable {
private TagPolicies tagPolicies;
private Map<String, SecurityZoneInfo> securityZones;
private List<RangerPolicyDelta> policyDeltas;

private Map<String, RangerPolicyDelta> deleteDeltas;
private Map<String, String> serviceConfig;

/**
Expand Down Expand Up @@ -183,14 +186,6 @@ public String toString() {

public void setPolicyDeltas(List<RangerPolicyDelta> policyDeltas) { this.policyDeltas = policyDeltas; }

public Map<String, RangerPolicyDelta> getDeleteDeltas() {
return deleteDeltas;
}

public void setDeleteDeltas(Map<String, RangerPolicyDelta> deleteDeltas) {
this.deleteDeltas = deleteDeltas;
}

@JsonInclude(JsonInclude.Include.NON_NULL)
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
Expand Down Expand Up @@ -417,7 +412,7 @@ public static ServicePolicies applyDelta(final ServicePolicies servicePolicies,
List<RangerPolicy> resourcePoliciesAfterDelete =
RangerPolicyDeltaUtil.deletePoliciesByDelta(oldResourcePolicies, deletedDeltaMap);
List<RangerPolicy> newResourcePolicies =
RangerPolicyDeltaUtil.applyDeltas(resourcePoliciesAfterDelete, servicePolicies.getPolicyDeltas(), servicePolicies.getServiceDef().getName());
RangerPolicyDeltaUtil.applyDeltas(resourcePoliciesAfterDelete, servicePolicies.getPolicyDeltas(), servicePolicies.getServiceDef().getName(), servicePolicies.getServiceName());

ret.setPolicies(newResourcePolicies);

Expand All @@ -428,7 +423,7 @@ public static ServicePolicies applyDelta(final ServicePolicies servicePolicies,
}
List<RangerPolicy> tagPoliciesAfterDelete =
RangerPolicyDeltaUtil.deletePoliciesByDelta(oldTagPolicies, deletedDeltaMap);
newTagPolicies = RangerPolicyDeltaUtil.applyDeltas(tagPoliciesAfterDelete, servicePolicies.getPolicyDeltas(), servicePolicies.getTagPolicies().getServiceDef().getName());
newTagPolicies = RangerPolicyDeltaUtil.applyDeltas(tagPoliciesAfterDelete, servicePolicies.getPolicyDeltas(), servicePolicies.getTagPolicies().getServiceDef().getName(), servicePolicies.getTagPolicies().getServiceName());
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("No need to apply deltas for tag policies");
Expand All @@ -444,38 +439,6 @@ public static ServicePolicies applyDelta(final ServicePolicies servicePolicies,
ret.getTagPolicies().setPolicies(newTagPolicies);
}

if (MapUtils.isNotEmpty(servicePolicies.getSecurityZones())) {
Map<String, SecurityZoneInfo> newSecurityZones = new HashMap<>();

for (Map.Entry<String, SecurityZoneInfo> entry : servicePolicies.getSecurityZones().entrySet()) {
String zoneName = entry.getKey();
SecurityZoneInfo zoneInfo = entry.getValue();

List<RangerPolicy> zoneResourcePolicies = policyEngine.getResourcePolicies(zoneName);
// There are no separate tag-policy-repositories for each zone

if (LOG.isDebugEnabled()) {
LOG.debug("Applying deltas for security-zone:[" + zoneName + "]");
}

final List<RangerPolicy> newZonePolicies = RangerPolicyDeltaUtil.applyDeltas(zoneResourcePolicies, zoneInfo.getPolicyDeltas(), servicePolicies.getServiceDef().getName());

if (LOG.isDebugEnabled()) {
LOG.debug("New resource policies for security-zone:[" + zoneName + "], zoneResourcePolicies:[" + Arrays.toString(newZonePolicies.toArray())+ "]");
}

SecurityZoneInfo newZoneInfo = new SecurityZoneInfo();

newZoneInfo.setZoneName(zoneName);
newZoneInfo.setResources(zoneInfo.getResources());
newZoneInfo.setPolicies(newZonePolicies);

newSecurityZones.put(zoneName, newZoneInfo);
}

ret.setSecurityZones(newSecurityZones);
}

return ret;
}
}
10 changes: 1 addition & 9 deletions ...s-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -701,17 +701,9 @@ private Map<String, Object> getMap(String key, Object value) {
private RangerPolicy getRangerPolicy(AtlasEntityHeader atlasPolicy, String serviceType) {
RangerPolicy policy = new RangerPolicy();

//policy.setId(atlasPolicy.getGuid());
policy.setName((String) atlasPolicy.getAttribute(QUALIFIED_NAME));
policy.setService((String) atlasPolicy.getAttribute(ATTR_POLICY_SERVICE_NAME));

// Adding atlas as serviceType for tag policies, as atlas_tag doesn't have all the resource available for evaluation
if (serviceType != null && serviceType.equals(TAG_RESOURCE_NAME) && policy.getService().equals("atlas")) {
policy.setServiceType("atlas");
} else {
policy.setServiceType(serviceType);
}

policy.setServiceType(serviceType);
policy.setGuid(atlasPolicy.getGuid());
policy.setCreatedBy(atlasPolicy.getCreatedBy());
policy.setCreateTime(atlasPolicy.getCreateTime());
Expand Down

0 comments on commit e44b399

Please sign in to comment.