Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DG-1709 | Set tag policies and policies based on service name instead of serviceType #3872

Merged
merged 2 commits into from
Dec 10, 2024
Merged

DG-1709 | Set tag policies and policies based on service name instead of serviceType #3872

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
2e50489
nit: few refactors and policy saving based name instead of types
sumandas0 Dec 10, 2024
0577ea1
nit: add exceptions in setpolicies incase of failure it will retry
sumandas0 Dec 10, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -428,7 +428,8 @@ public void setPolicies(ServicePolicies policies) {
}

} catch (Exception e) {
LOG.error("setPolicies: policy engine initialization failed! Leaving current policy engine as-is. Exception : ", e);
LOG.error("setPolicies: Failed to set policies, didn't set policies", e);
throw e;
}
if (LOG.isDebugEnabled()) {
LOG.debug("<== setPolicies(" + policies + ")");
Expand Down
21 changes: 10 additions & 11 deletions auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,6 @@
import java.io.FileWriter;
import java.io.Reader;
import java.io.Writer;
import java.util.Date;
import java.util.Timer;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
Expand All @@ -62,7 +61,7 @@ public class PolicyRefresher extends Thread {
private final BlockingQueue<DownloadTrigger> policyDownloadQueue = new LinkedBlockingQueue<>();
private Timer policyDownloadTimer;
private long lastKnownVersion = -1L;
private long lastUpdatedTiemInMillis = -1L;
private long lastUpdatedTimeInMillis = -1L;
private long lastActivationTimeInMillis;
private boolean policiesSetInPlugin;
private boolean serviceDefSetInPlugin;
Expand Down Expand Up @@ -222,7 +221,7 @@ public void run() {
loadPolicy();
loadUserStore();
} catch(InterruptedException excp) {
LOG.info("PolicyRefresher(serviceName=" + serviceName + ").run(): interrupted! Exiting thread", excp);
LOG.info("PolicyRefreshxer(serviceName=" + serviceName + ").run(): interrupted! Exiting thread", excp);
break;
} finally {
if (trigger != null) {
Expand Down Expand Up @@ -279,7 +278,7 @@ private void loadPolicy() {
serviceDefSetInPlugin = false;
setLastActivationTimeInMillis(System.currentTimeMillis());
lastKnownVersion = svcPolicies.getPolicyVersion() != null ? svcPolicies.getPolicyVersion() : -1L;
lastUpdatedTiemInMillis = svcPolicies.getPolicyUpdateTime() != null ? svcPolicies.getPolicyUpdateTime().getTime() : -1L;
lastUpdatedTimeInMillis = svcPolicies.getPolicyUpdateTime() != null ? svcPolicies.getPolicyUpdateTime().getTime() : -1L;
} else {
if (!policiesSetInPlugin && !serviceDefSetInPlugin) {
plugIn.setPolicies(null);
Expand All @@ -293,10 +292,10 @@ private void loadPolicy() {
serviceDefSetInPlugin = true;
setLastActivationTimeInMillis(System.currentTimeMillis());
lastKnownVersion = -1;
lastUpdatedTiemInMillis = -1;
lastUpdatedTimeInMillis = -1;
}
} catch (Exception excp) {
LOG.error("Encountered unexpected exception, ignoring..", excp);
LOG.error("Encountered unexpected exception!!!!!!!!!!!", excp);
}

RangerPerfTracer.log(perf);
Expand All @@ -323,17 +322,17 @@ private ServicePolicies loadPolicyfromPolicyAdmin() throws RangerServiceNotFound
try {


if (serviceName.equals("atlas") && plugIn.getTypeRegistry() != null && lastUpdatedTiemInMillis == -1) {
if (serviceName.equals("atlas") && plugIn.getTypeRegistry() != null && lastUpdatedTimeInMillis == -1) {
LOG.info("PolicyRefresher(serviceName=" + serviceName + "): loading all policies for first time");
RangerRESTUtils restUtils = new RangerRESTUtils();
CachePolicyTransformerImpl transformer = new CachePolicyTransformerImpl(plugIn.getTypeRegistry());

svcPolicies = transformer.getPoliciesAll(serviceName,
restUtils.getPluginId(serviceName, plugIn.getAppId()),
lastUpdatedTiemInMillis);
lastUpdatedTimeInMillis);
} else {
LOG.info("PolicyRefresher(serviceName=" + serviceName + "): loading delta policies from last known version=" + lastKnownVersion + ", lastUpdatedTime=" + lastUpdatedTiemInMillis);
svcPolicies = atlasAuthAdminClient.getServicePoliciesIfUpdated(lastUpdatedTiemInMillis, this.enableDeltaBasedRefresh);
LOG.info("PolicyRefresher(serviceName=" + serviceName + "): loading delta policies from last known version=" + lastKnownVersion + ", lastUpdatedTime=" + lastUpdatedTimeInMillis);
svcPolicies = atlasAuthAdminClient.getServicePoliciesIfUpdated(lastUpdatedTimeInMillis, this.enableDeltaBasedRefresh);
}

boolean isUpdated = svcPolicies != null;
Expand Down Expand Up @@ -400,7 +399,7 @@ private ServicePolicies loadFromCache() {
}

lastKnownVersion = policies.getPolicyVersion() == null ? -1 : policies.getPolicyVersion().longValue();
lastUpdatedTiemInMillis = policies.getPolicyUpdateTime() == null ? -1 : policies.getPolicyUpdateTime().getTime();
lastUpdatedTimeInMillis = policies.getPolicyUpdateTime() == null ? -1 : policies.getPolicyUpdateTime().getTime();
}
} catch (Exception excp) {
LOG.error("failed to load policies from cache file " + cacheFile.getAbsolutePath(), excp);
Expand Down
10 changes: 5 additions & 5 deletions auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ public static List<RangerPolicy> deletePoliciesByDelta(List<RangerPolicy> polici
}
}

public static List<RangerPolicy> applyDeltas(List<RangerPolicy> policies, List<RangerPolicyDelta> deltas, String serviceType) {
public static List<RangerPolicy> applyDeltas(List<RangerPolicy> policies, List<RangerPolicyDelta> deltas, String serviceType, String serviceName) {
if (LOG.isDebugEnabled()) {
LOG.debug("==> applyDeltas(serviceType=" + serviceType + ")");
}
Expand All @@ -61,9 +61,9 @@ public static List<RangerPolicy> applyDeltas(List<RangerPolicy> policies, List<R
return policies;
}

boolean hasExpectedServiceType = deltas.stream().anyMatch(delta -> serviceType.equals(delta.getServiceType()));
boolean hasExpectedServiceName = deltas.stream().anyMatch(delta -> serviceName.equals(delta.getPolicy().getService()));

if (!hasExpectedServiceType) {
if (!hasExpectedServiceName) {
if (LOG.isDebugEnabled()) {
LOG.debug("No deltas match the expected serviceType: " + serviceType);
}
Expand All @@ -73,7 +73,7 @@ public static List<RangerPolicy> applyDeltas(List<RangerPolicy> policies, List<R
List<RangerPolicy> updatedPolicies = new ArrayList<>(policies);

for (RangerPolicyDelta delta : deltas) {
if (!serviceType.equals(delta.getServiceType())) {
if (!serviceName.equals(delta.getPolicy().getService())) {
continue;
}

Expand All @@ -83,7 +83,7 @@ public static List<RangerPolicy> applyDeltas(List<RangerPolicy> policies, List<R
updatedPolicies.add(delta.getPolicy());
break;
default:
LOG.warn("Unexpected changeType in policyDelta: [" + delta + "]. Ignoring delta.");
LOG.warn("Unexpected changeType in policyDelta: [" + delta.getPolicyGuid() + "]. Ignoring delta.");
}
}

Expand Down
53 changes: 8 additions & 45 deletions auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,12 @@
import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlRootElement;
import java.util.*;
import java.util.Arrays;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

@JsonInclude(JsonInclude.Include.NON_NULL)
@XmlRootElement
Expand All @@ -53,8 +58,6 @@ public class ServicePolicies implements java.io.Serializable {
private TagPolicies tagPolicies;
private Map<String, SecurityZoneInfo> securityZones;
private List<RangerPolicyDelta> policyDeltas;

private Map<String, RangerPolicyDelta> deleteDeltas;
private Map<String, String> serviceConfig;

/**
Expand Down Expand Up @@ -183,14 +186,6 @@ public String toString() {

public void setPolicyDeltas(List<RangerPolicyDelta> policyDeltas) { this.policyDeltas = policyDeltas; }

public Map<String, RangerPolicyDelta> getDeleteDeltas() {
return deleteDeltas;
}

public void setDeleteDeltas(Map<String, RangerPolicyDelta> deleteDeltas) {
this.deleteDeltas = deleteDeltas;
}

@JsonInclude(JsonInclude.Include.NON_NULL)
@XmlRootElement
@XmlAccessorType(XmlAccessType.FIELD)
Expand Down Expand Up @@ -417,7 +412,7 @@ public static ServicePolicies applyDelta(final ServicePolicies servicePolicies,
List<RangerPolicy> resourcePoliciesAfterDelete =
RangerPolicyDeltaUtil.deletePoliciesByDelta(oldResourcePolicies, deletedDeltaMap);
List<RangerPolicy> newResourcePolicies =
RangerPolicyDeltaUtil.applyDeltas(resourcePoliciesAfterDelete, servicePolicies.getPolicyDeltas(), servicePolicies.getServiceDef().getName());
RangerPolicyDeltaUtil.applyDeltas(resourcePoliciesAfterDelete, servicePolicies.getPolicyDeltas(), servicePolicies.getServiceDef().getName(), servicePolicies.getServiceName());

ret.setPolicies(newResourcePolicies);

Expand All @@ -428,7 +423,7 @@ public static ServicePolicies applyDelta(final ServicePolicies servicePolicies,
}
List<RangerPolicy> tagPoliciesAfterDelete =
RangerPolicyDeltaUtil.deletePoliciesByDelta(oldTagPolicies, deletedDeltaMap);
newTagPolicies = RangerPolicyDeltaUtil.applyDeltas(tagPoliciesAfterDelete, servicePolicies.getPolicyDeltas(), servicePolicies.getTagPolicies().getServiceDef().getName());
newTagPolicies = RangerPolicyDeltaUtil.applyDeltas(tagPoliciesAfterDelete, servicePolicies.getPolicyDeltas(), servicePolicies.getTagPolicies().getServiceDef().getName(), servicePolicies.getTagPolicies().getServiceName());
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("No need to apply deltas for tag policies");
Expand All @@ -444,38 +439,6 @@ public static ServicePolicies applyDelta(final ServicePolicies servicePolicies,
ret.getTagPolicies().setPolicies(newTagPolicies);
}

if (MapUtils.isNotEmpty(servicePolicies.getSecurityZones())) {
Map<String, SecurityZoneInfo> newSecurityZones = new HashMap<>();

for (Map.Entry<String, SecurityZoneInfo> entry : servicePolicies.getSecurityZones().entrySet()) {
String zoneName = entry.getKey();
SecurityZoneInfo zoneInfo = entry.getValue();

List<RangerPolicy> zoneResourcePolicies = policyEngine.getResourcePolicies(zoneName);
// There are no separate tag-policy-repositories for each zone

if (LOG.isDebugEnabled()) {
LOG.debug("Applying deltas for security-zone:[" + zoneName + "]");
}

final List<RangerPolicy> newZonePolicies = RangerPolicyDeltaUtil.applyDeltas(zoneResourcePolicies, zoneInfo.getPolicyDeltas(), servicePolicies.getServiceDef().getName());

if (LOG.isDebugEnabled()) {
LOG.debug("New resource policies for security-zone:[" + zoneName + "], zoneResourcePolicies:[" + Arrays.toString(newZonePolicies.toArray())+ "]");
}

SecurityZoneInfo newZoneInfo = new SecurityZoneInfo();

newZoneInfo.setZoneName(zoneName);
newZoneInfo.setResources(zoneInfo.getResources());
newZoneInfo.setPolicies(newZonePolicies);

newSecurityZones.put(zoneName, newZoneInfo);
}

ret.setSecurityZones(newSecurityZones);
}

return ret;
}
}
10 changes: 1 addition & 9 deletions ...s-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -708,17 +708,9 @@ private Map<String, Object> getMap(String key, Object value) {
private RangerPolicy getRangerPolicy(AtlasEntityHeader atlasPolicy, String serviceType) {
RangerPolicy policy = new RangerPolicy();

//policy.setId(atlasPolicy.getGuid());
policy.setName((String) atlasPolicy.getAttribute(QUALIFIED_NAME));
policy.setService((String) atlasPolicy.getAttribute(ATTR_POLICY_SERVICE_NAME));

// Adding atlas as serviceType for tag policies, as atlas_tag doesn't have all the resource available for evaluation
if (serviceType != null && serviceType.equals(TAG_RESOURCE_NAME) && policy.getService().equals("atlas")) {
policy.setServiceType("atlas");
} else {
policy.setServiceType(serviceType);
}

policy.setServiceType(serviceType);
policy.setGuid(atlasPolicy.getGuid());
policy.setCreatedBy(atlasPolicy.getCreatedBy());
policy.setCreateTime(atlasPolicy.getCreateTime());
Expand Down
Loading