diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java new file mode 100644 index 0000000000..64b46c3c95 --- /dev/null +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicy.java @@ -0,0 +1,1704 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.atlas.plugin.model; + +import org.apache.commons.collections.CollectionUtils; +import org.apache.htrace.shaded.fasterxml.jackson.annotation.JsonInclude; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + + +@JsonInclude(JsonInclude.Include.NON_NULL) +@XmlRootElement +@XmlAccessorType(XmlAccessType.FIELD) +public class RangerPolicy extends RangerBaseModelObject implements java.io.Serializable { + public static final String POLICY_TYPE_ACCESS = "ACCESS"; + public static final String POLICY_TYPE_DATAMASK = "DATA_MASK"; + public static final String POLICY_TYPE_ROWFILTER = "ROW_FILTER"; + public static final String POLICY_TYPE_AUDIT = "AUDIT"; + + public static final String[] POLICY_TYPES = new String[] { + POLICY_TYPE_ACCESS, + POLICY_TYPE_DATAMASK, + POLICY_TYPE_ROWFILTER + }; + + public static final String MASK_TYPE_NULL = "MASK_NULL"; + public static final String MASK_TYPE_NONE = "MASK_NONE"; + public static final String MASK_TYPE_CUSTOM = "CUSTOM"; + + public static final int POLICY_PRIORITY_NORMAL = 0; + public static final int POLICY_PRIORITY_OVERRIDE = 1; + + public static final String POLICY_PRIORITY_NAME_NORMAL = "NORMAL"; + public static final String POLICY_PRIORITY_NAME_OVERRIDE = "OVERRIDE"; + + public static final Comparator POLICY_ID_COMPARATOR = new PolicyIdComparator(); + + // For future use + private static final long serialVersionUID = 1L; + + private String service; + private String name; + private String policyType; + private Integer policyPriority; + private String description; + private String resourceSignature; + private Boolean isAuditEnabled; + private Map resources; + private List conditions; + private List policyItems; + private List denyPolicyItems; + private List allowExceptions; + private List denyExceptions; + private List dataMaskPolicyItems; + private List rowFilterPolicyItems; + private String serviceType; + private Map options; + private List validitySchedules; + private List policyLabels; + private String zoneName; + private Boolean isDenyAllElse; + private Map attributes; + + public RangerPolicy() { + this(null, null, null, null, null, null, null, null, null, null, null); + } + + public RangerPolicy(String service, String name, String policyType, Integer policyPriority, String description, Map resources, List policyItems, String resourceSignature, Map options, List validitySchedules, List policyLables) { + this(service, name, policyType, policyPriority, description, resources, policyItems, resourceSignature, options, validitySchedules, policyLables, null); + } + + public RangerPolicy(String service, String name, String policyType, Integer policyPriority, String description, Map resources, List policyItems, String resourceSignature, Map options, List validitySchedules, List policyLables, String zoneName) { + this(service, name, policyType, policyPriority, description, resources, policyItems, resourceSignature, options, validitySchedules, policyLables, zoneName, null); + } + + public RangerPolicy(String service, String name, String policyType, Integer policyPriority, String description, Map resources, List policyItems, String resourceSignature, Map options, List validitySchedules, List policyLables, String zoneName, List conditions) { + this(service, name, policyType, policyPriority, description, resources, policyItems, resourceSignature, options, validitySchedules, policyLables, zoneName, conditions, null); + } + + /** + * @param service + * @param name + * @param policyType + * @param description + * @param resources + * @param policyItems + * @param resourceSignature TODO + */ + public RangerPolicy(String service, String name, String policyType, Integer policyPriority, String description, Map resources, List policyItems, String resourceSignature, Map options, List validitySchedules, List policyLables, String zoneName, List conditions, Boolean isDenyAllElse) { + super(); + + setService(service); + setName(name); + setPolicyType(policyType); + setPolicyPriority(policyPriority); + setDescription(description); + setResourceSignature(resourceSignature); + setIsAuditEnabled(null); + setResources(resources); + setPolicyItems(policyItems); + setDenyPolicyItems(null); + setAllowExceptions(null); + setDenyExceptions(null); + setDataMaskPolicyItems(null); + setRowFilterPolicyItems(null); + setOptions(options); + setValiditySchedules(validitySchedules); + setPolicyLabels(policyLables); + setZoneName(zoneName); + setConditions(conditions); + setIsDenyAllElse(isDenyAllElse); + + } + + /** + * @param other + */ + public void updateFrom(RangerPolicy other) { + super.updateFrom(other); + + setService(other.getService()); + setName(other.getName()); + setPolicyType(other.getPolicyType()); + setPolicyPriority(other.getPolicyPriority()); + setDescription(other.getDescription()); + setResourceSignature(other.getResourceSignature()); + setIsAuditEnabled(other.getIsAuditEnabled()); + setResources(other.getResources()); + setConditions(other.getConditions()); + setPolicyItems(other.getPolicyItems()); + setDenyPolicyItems(other.getDenyPolicyItems()); + setAllowExceptions(other.getAllowExceptions()); + setDenyExceptions(other.getDenyExceptions()); + setDataMaskPolicyItems(other.getDataMaskPolicyItems()); + setRowFilterPolicyItems(other.getRowFilterPolicyItems()); + setServiceType(other.getServiceType()); + setOptions(other.getOptions()); + setValiditySchedules(other.getValiditySchedules()); + setPolicyLabels(other.getPolicyLabels()); + setZoneName(other.getZoneName()); + setIsDenyAllElse(other.getIsDenyAllElse()); + } + + public Map getAttributes() { + return attributes; + } + + public void setAttributes(Map attributes) { + this.attributes = attributes; + } + + /** + * @return the type + */ + public String getService() { + return service; + } + + /** + * @param service the type to set + */ + public void setService(String service) { + this.service = service; + } + + /** + * @return the name + */ + public String getName() { + return name; + } + + /** + * @param name the name to set + */ + public void setName(String name) { + this.name = name; + } + + /** + * @return the policyType + */ + public String getPolicyType() { + return policyType; + } + + /** + * @param policyType the policyType to set + */ + public void setPolicyType(String policyType) { + this.policyType = policyType; + } + + /** + * @return the policyPriority + */ + public Integer getPolicyPriority() { + return policyPriority; + } + + /** + * @param policyPriority the policyPriority to set + */ + public void setPolicyPriority(Integer policyPriority) { + this.policyPriority = policyPriority == null ? RangerPolicy.POLICY_PRIORITY_NORMAL : policyPriority; + } + + /** + * @return the description + */ + public String getDescription() { + return description; + } + + /** + * @param description the description to set + */ + public void setDescription(String description) { + this.description = description; + } + + /** + * @return the resourceSignature + */ + public String getResourceSignature() { + return resourceSignature; + } + + /** + * @param resourceSignature the resourceSignature to set + */ + public void setResourceSignature(String resourceSignature) { + this.resourceSignature = resourceSignature; + } + + /** + * @return the isAuditEnabled + */ + public Boolean getIsAuditEnabled() { + return isAuditEnabled; + } + + /** + * @param isAuditEnabled the isEnabled to set + */ + public void setIsAuditEnabled(Boolean isAuditEnabled) { + this.isAuditEnabled = isAuditEnabled == null ? Boolean.TRUE : isAuditEnabled; + } + + public String getServiceType() { + return serviceType; + } + + public void setServiceType(String serviceType) { + this.serviceType = serviceType; + } + + public List getPolicyLabels() { + return policyLabels; + } + + public void setPolicyLabels(List policyLabels) { + if (this.policyLabels == null) { + this.policyLabels = new ArrayList<>(); + } + + if (this.policyLabels == policyLabels) { + return; + } + + this.policyLabels.clear(); + + if (policyLabels != null) { + this.policyLabels.addAll(policyLabels); + } + } + + /** + * @return the resources + */ + public Map getResources() { + return resources; + } + + /** + * @param resources the resources to set + */ + public void setResources(Map resources) { + if(this.resources == null) { + this.resources = new HashMap<>(); + } + + if(this.resources == resources) { + return; + } + + this.resources.clear(); + + if(resources != null) { + for(Map.Entry e : resources.entrySet()) { + this.resources.put(e.getKey(), e.getValue()); + } + } + } + + /** + * @return the policyItems + */ + public List getPolicyItems() { + return policyItems; + } + + /** + * @param policyItems the policyItems to set + */ + public void setPolicyItems(List policyItems) { + if(this.policyItems == null) { + this.policyItems = new ArrayList<>(); + } + + if(this.policyItems == policyItems) { + return; + } + + this.policyItems.clear(); + + if(policyItems != null) { + this.policyItems.addAll(policyItems); + } + } + + /** + * @return the denyPolicyItems + */ + public List getDenyPolicyItems() { + return denyPolicyItems; + } + + /** + * @param denyPolicyItems the denyPolicyItems to set + */ + public void setDenyPolicyItems(List denyPolicyItems) { + if(this.denyPolicyItems == null) { + this.denyPolicyItems = new ArrayList<>(); + } + + if(this.denyPolicyItems == denyPolicyItems) { + return; + } + + this.denyPolicyItems.clear(); + + if(denyPolicyItems != null) { + this.denyPolicyItems.addAll(denyPolicyItems); + } + } + + /** + * @return the allowExceptions + */ + public List getAllowExceptions() { + return allowExceptions; + } + + /** + * @param allowExceptions the allowExceptions to set + */ + public void setAllowExceptions(List allowExceptions) { + if(this.allowExceptions == null) { + this.allowExceptions = new ArrayList<>(); + } + + if(this.allowExceptions == allowExceptions) { + return; + } + + this.allowExceptions.clear(); + + if(allowExceptions != null) { + this.allowExceptions.addAll(allowExceptions); + } + } + + /** + * @return the denyExceptions + */ + public List getDenyExceptions() { + return denyExceptions; + } + + /** + * @param denyExceptions the denyExceptions to set + */ + public void setDenyExceptions(List denyExceptions) { + if(this.denyExceptions == null) { + this.denyExceptions = new ArrayList<>(); + } + + if(this.denyExceptions == denyExceptions) { + return; + } + + this.denyExceptions.clear(); + + if(denyExceptions != null) { + this.denyExceptions.addAll(denyExceptions); + } + } + + public List getDataMaskPolicyItems() { + return dataMaskPolicyItems; + } + + public void setDataMaskPolicyItems(List dataMaskPolicyItems) { + if(this.dataMaskPolicyItems == null) { + this.dataMaskPolicyItems = new ArrayList<>(); + } + + if(this.dataMaskPolicyItems == dataMaskPolicyItems) { + return; + } + + this.dataMaskPolicyItems.clear(); + + if(dataMaskPolicyItems != null) { + this.dataMaskPolicyItems.addAll(dataMaskPolicyItems); + } + } + + public List getRowFilterPolicyItems() { + return rowFilterPolicyItems; + } + + public void setRowFilterPolicyItems(List rowFilterPolicyItems) { + if(this.rowFilterPolicyItems == null) { + this.rowFilterPolicyItems = new ArrayList<>(); + } + + if(this.rowFilterPolicyItems == rowFilterPolicyItems) { + return; + } + + this.rowFilterPolicyItems.clear(); + + if(rowFilterPolicyItems != null) { + this.rowFilterPolicyItems.addAll(rowFilterPolicyItems); + } + } + + public Map getOptions() { return options; } + + public void setOptions(Map options) { + if (this.options == null) { + this.options = new HashMap<>(); + } + if (this.options == options) { + return; + } + this.options.clear(); + + if(options != null) { + for(Map.Entry e : options.entrySet()) { + this.options.put(e.getKey(), e.getValue()); + } + } + } + + public List getValiditySchedules() { return validitySchedules; } + + public void setValiditySchedules(List validitySchedules) { + if (this.validitySchedules == null) { + this.validitySchedules = new ArrayList<>(); + } + if (this.validitySchedules == validitySchedules) { + return; + } + this.validitySchedules.clear(); + + if(validitySchedules != null) { + this.validitySchedules.addAll(validitySchedules); + } + } + public String getZoneName() { return zoneName; } + + public void setZoneName(String zoneName) { + this.zoneName = zoneName; + } + + /** + * @return the conditions + */ + public List getConditions() { return conditions; } + /** + * @param conditions the conditions to set + */ + public void setConditions(List conditions) { + this.conditions = conditions; + } + + public Boolean getIsDenyAllElse() { + return isDenyAllElse; + } + + public void setIsDenyAllElse(Boolean isDenyAllElse) { + this.isDenyAllElse = isDenyAllElse == null ? Boolean.FALSE : isDenyAllElse; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerPolicy={"); + + super.toString(sb); + + sb.append("service={").append(service).append("} "); + sb.append("name={").append(name).append("} "); + sb.append("policyType={").append(policyType).append("} "); + sb.append("policyPriority={").append(policyPriority).append("} "); + sb.append("description={").append(description).append("} "); + sb.append("resourceSignature={").append(resourceSignature).append("} "); + sb.append("isAuditEnabled={").append(isAuditEnabled).append("} "); + sb.append("serviceType={").append(serviceType).append("} "); + + sb.append("resources={"); + if(resources != null) { + for(Map.Entry e : resources.entrySet()) { + sb.append(e.getKey()).append("={"); + e.getValue().toString(sb); + sb.append("} "); + } + } + sb.append("} "); + sb.append("policyLabels={"); + if(policyLabels != null) { + for(String policyLabel : policyLabels) { + if(policyLabel != null) { + sb.append(policyLabel).append(" "); + } + } + } + sb.append("} "); + + sb.append("policyConditions={"); + if(conditions != null) { + for(RangerPolicyItemCondition condition : conditions) { + if(condition != null) { + condition.toString(sb); + } + } + } + sb.append("} "); + + sb.append("policyItems={"); + if(policyItems != null) { + for(RangerPolicyItem policyItem : policyItems) { + if(policyItem != null) { + policyItem.toString(sb); + } + } + } + sb.append("} "); + + sb.append("denyPolicyItems={"); + if(denyPolicyItems != null) { + for(RangerPolicyItem policyItem : denyPolicyItems) { + if(policyItem != null) { + policyItem.toString(sb); + } + } + } + sb.append("} "); + + sb.append("allowExceptions={"); + if(allowExceptions != null) { + for(RangerPolicyItem policyItem : allowExceptions) { + if(policyItem != null) { + policyItem.toString(sb); + } + } + } + sb.append("} "); + + sb.append("denyExceptions={"); + if(denyExceptions != null) { + for(RangerPolicyItem policyItem : denyExceptions) { + if(policyItem != null) { + policyItem.toString(sb); + } + } + } + sb.append("} "); + + sb.append("dataMaskPolicyItems={"); + if(dataMaskPolicyItems != null) { + for(RangerDataMaskPolicyItem dataMaskPolicyItem : dataMaskPolicyItems) { + if(dataMaskPolicyItem != null) { + dataMaskPolicyItem.toString(sb); + } + } + } + sb.append("} "); + + sb.append("rowFilterPolicyItems={"); + if(rowFilterPolicyItems != null) { + for(RangerRowFilterPolicyItem rowFilterPolicyItem : rowFilterPolicyItems) { + if(rowFilterPolicyItem != null) { + rowFilterPolicyItem.toString(sb); + } + } + } + sb.append("} "); + + sb.append("options={"); + if(options != null) { + for(Map.Entry e : options.entrySet()) { + sb.append(e.getKey()).append("={"); + sb.append(e.getValue().toString()); + sb.append("} "); + } + } + sb.append("} "); + + //sb.append("validitySchedules={").append(validitySchedules).append("} "); + sb.append("validitySchedules={"); + if (CollectionUtils.isNotEmpty(validitySchedules)) { + for (RangerValiditySchedule schedule : validitySchedules) { + if (schedule != null) { + sb.append("schedule={").append(schedule).append("}"); + } + } + } + sb.append(", zoneName=").append(zoneName); + + sb.append(", isDenyAllElse={").append(isDenyAllElse).append("} "); + + sb.append("}"); + + sb.append("}"); + + return sb; + } + + static class PolicyIdComparator implements Comparator, java.io.Serializable { + @Override + public int compare(RangerPolicy me, RangerPolicy other) { + return Long.compare(me.getId(), other.getId()); +// return String.CASE_INSENSITIVE_ORDER.compare(me.getGuid(), other.getGuid()); + } + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class RangerPolicyResource implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private List values; + private Boolean isExcludes; + private Boolean isRecursive; + + public RangerPolicyResource() { + this((List)null, null, null); + } + + public RangerPolicyResource(String value) { + setValue(value); + setIsExcludes(null); + setIsRecursive(null); + } + + public RangerPolicyResource(String value, Boolean isExcludes, Boolean isRecursive) { + setValue(value); + setIsExcludes(isExcludes); + setIsRecursive(isRecursive); + } + + public RangerPolicyResource(List values, Boolean isExcludes, Boolean isRecursive) { + setValues(values); + setIsExcludes(isExcludes); + setIsRecursive(isRecursive); + } + + /** + * @return the values + */ + public List getValues() { + return values; + } + + /** + * @param values the values to set + */ + public void setValues(List values) { + if(this.values == null) { + this.values = new ArrayList<>(); + } + + if(this.values == values) { + return; + } + + this.values.clear(); + + if(values != null) { + this.values.addAll(values); + } + } + + /** + * @param value the value to set + */ + public void setValue(String value) { + if(this.values == null) { + this.values = new ArrayList<>(); + } + + this.values.clear(); + + this.values.add(value); + } + + /** + * @return the isExcludes + */ + public Boolean getIsExcludes() { + return isExcludes; + } + + /** + * @param isExcludes the isExcludes to set + */ + public void setIsExcludes(Boolean isExcludes) { + this.isExcludes = isExcludes == null ? Boolean.FALSE : isExcludes; + } + + /** + * @return the isRecursive + */ + public Boolean getIsRecursive() { + return isRecursive; + } + + /** + * @param isRecursive the isRecursive to set + */ + public void setIsRecursive(Boolean isRecursive) { + this.isRecursive = isRecursive == null ? Boolean.FALSE : isRecursive; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerPolicyResource={"); + sb.append("values={"); + if(values != null) { + for(String value : values) { + sb.append(value).append(" "); + } + } + sb.append("} "); + sb.append("isExcludes={").append(isExcludes).append("} "); + sb.append("isRecursive={").append(isRecursive).append("} "); + sb.append("}"); + + return sb; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + + ((isExcludes == null) ? 0 : isExcludes.hashCode()); + result = prime * result + + ((isRecursive == null) ? 0 : isRecursive.hashCode()); + result = prime * result + + ((values == null) ? 0 : values.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + RangerPolicyResource other = (RangerPolicyResource) obj; + if (isExcludes == null) { + if (other.isExcludes != null) + return false; + } else if (!isExcludes.equals(other.isExcludes)) + return false; + if (isRecursive == null) { + if (other.isRecursive != null) + return false; + } else if (!isRecursive.equals(other.isRecursive)) + return false; + if (values == null) { + if (other.values != null) + return false; + } else if (!values.equals(other.values)) + return false; + return true; + } + + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class RangerPolicyItem implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private List accesses; + private List users; + private List groups; + private List roles; + private List conditions; + private Boolean delegateAdmin; + + public RangerPolicyItem() { + this(null, null, null, null, null, null); + } + + public RangerPolicyItem(List accessTypes, List users, List groups, List roles, List conditions, Boolean delegateAdmin) { + setAccesses(accessTypes); + setUsers(users); + setGroups(groups); + setRoles(roles); + setConditions(conditions); + setDelegateAdmin(delegateAdmin); + } + + /** + * @return the accesses + */ + public List getAccesses() { + return accesses; + } + /** + * @param accesses the accesses to set + */ + public void setAccesses(List accesses) { + if(this.accesses == null) { + this.accesses = new ArrayList<>(); + } + + if(this.accesses == accesses) { + return; + } + + this.accesses.clear(); + + if(accesses != null) { + this.accesses.addAll(accesses); + } + } + /** + * @return the users + */ + public List getUsers() { + return users; + } + /** + * @param users the users to set + */ + public void setUsers(List users) { + if(this.users == null) { + this.users = new ArrayList<>(); + } + + if(this.users == users) { + return; + } + + this.users.clear(); + + if(users != null) { + this.users.addAll(users); + } + } + /** + * @return the groups + */ + public List getGroups() { + return groups; + } + /** + * @param groups the groups to set + */ + public void setGroups(List groups) { + if(this.groups == null) { + this.groups = new ArrayList<>(); + } + + if(this.groups == groups) { + return; + } + + this.groups.clear(); + + if(groups != null) { + this.groups.addAll(groups); + } + } + /** + * @return the roles + */ + public List getRoles() { + return roles; + } + /** + * @param roles the roles to set + */ + public void setRoles(List roles) { + if(this.roles == null) { + this.roles = new ArrayList<>(); + } + + if(this.roles == roles) { + return; + } + + this.roles.clear(); + + if(roles != null) { + this.roles.addAll(roles); + } + } + /** + * @return the conditions + */ + public List getConditions() { + return conditions; + } + /** + * @param conditions the conditions to set + */ + public void setConditions(List conditions) { + if(this.conditions == null) { + this.conditions = new ArrayList<>(); + } + + if(this.conditions == conditions) { + return; + } + + this.conditions.clear(); + + if(conditions != null) { + this.conditions.addAll(conditions); + } + } + + /** + * @return the delegateAdmin + */ + public Boolean getDelegateAdmin() { + return delegateAdmin; + } + + /** + * @param delegateAdmin the delegateAdmin to set + */ + public void setDelegateAdmin(Boolean delegateAdmin) { + this.delegateAdmin = delegateAdmin == null ? Boolean.FALSE : delegateAdmin; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerPolicyItem={"); + + sb.append("accessTypes={"); + if(accesses != null) { + for(RangerPolicyItemAccess access : accesses) { + if(access != null) { + access.toString(sb); + } + } + } + sb.append("} "); + + sb.append("users={"); + if(users != null) { + for(String user : users) { + if(user != null) { + sb.append(user).append(" "); + } + } + } + sb.append("} "); + + sb.append("groups={"); + if(groups != null) { + for(String group : groups) { + if(group != null) { + sb.append(group).append(" "); + } + } + } + sb.append("} "); + + sb.append("roles={"); + if(roles != null) { + for(String role : roles) { + if(role != null) { + sb.append(role).append(" "); + } + } + } + sb.append("} "); + + sb.append("conditions={"); + if(conditions != null) { + for(RangerPolicyItemCondition condition : conditions) { + if(condition != null) { + condition.toString(sb); + } + } + } + sb.append("} "); + + sb.append("delegateAdmin={").append(delegateAdmin).append("} "); + sb.append("}"); + + return sb; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + + ((accesses == null) ? 0 : accesses.hashCode()); + result = prime * result + + ((conditions == null) ? 0 : conditions.hashCode()); + result = prime * result + + ((delegateAdmin == null) ? 0 : delegateAdmin.hashCode()); + result = prime * result + + ((roles == null) ? 0 : roles.hashCode()); + result = prime * result + + ((groups == null) ? 0 : groups.hashCode()); + result = prime * result + ((users == null) ? 0 : users.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + RangerPolicyItem other = (RangerPolicyItem) obj; + if (accesses == null) { + if (other.accesses != null) + return false; + } else if (!accesses.equals(other.accesses)) + return false; + if (conditions == null) { + if (other.conditions != null) + return false; + } else if (!conditions.equals(other.conditions)) + return false; + if (delegateAdmin == null) { + if (other.delegateAdmin != null) + return false; + } else if (!delegateAdmin.equals(other.delegateAdmin)) + return false; + if (roles == null) { + if (other.roles != null) + return false; + } else if (!roles.equals(other.roles)) + return false; + if (groups == null) { + if (other.groups != null) + return false; + } else if (!groups.equals(other.groups)) + return false; + if (users == null) { + if (other.users != null) + return false; + } else if (!users.equals(other.users)) + return false; + return true; + + } + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class RangerDataMaskPolicyItem extends RangerPolicyItem implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private RangerPolicyItemDataMaskInfo dataMaskInfo; + + public RangerDataMaskPolicyItem() { + this(null, null, null, null, null, null, null); + } + + public RangerDataMaskPolicyItem(List accesses, RangerPolicyItemDataMaskInfo dataMaskDetail, List users, List groups, List roles, List conditions, Boolean delegateAdmin) { + super(accesses, users, groups, roles, conditions, delegateAdmin); + + setDataMaskInfo(dataMaskDetail); + } + + /** + * @return the dataMaskInfo + */ + public RangerPolicyItemDataMaskInfo getDataMaskInfo() { + return dataMaskInfo; + } + + /** + * @param dataMaskInfo the dataMaskInfo to set + */ + public void setDataMaskInfo(RangerPolicyItemDataMaskInfo dataMaskInfo) { + this.dataMaskInfo = dataMaskInfo == null ? new RangerPolicyItemDataMaskInfo() : dataMaskInfo; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = super.hashCode(); + result = prime * result + ((dataMaskInfo == null) ? 0 : dataMaskInfo.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if(! super.equals(obj)) + return false; + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + RangerDataMaskPolicyItem other = (RangerDataMaskPolicyItem) obj; + if (dataMaskInfo == null) { + if (other.dataMaskInfo != null) + return false; + } else if (!dataMaskInfo.equals(other.dataMaskInfo)) + return false; + return true; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerDataMaskPolicyItem={"); + + super.toString(sb); + + sb.append("dataMaskInfo={"); + if(dataMaskInfo != null) { + dataMaskInfo.toString(sb); + } + sb.append("} "); + + sb.append("}"); + + return sb; + } + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class RangerRowFilterPolicyItem extends RangerPolicyItem implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private RangerPolicyItemRowFilterInfo rowFilterInfo; + + public RangerRowFilterPolicyItem() { + this(null, null, null, null, null, null, null); + } + + public RangerRowFilterPolicyItem(RangerPolicyItemRowFilterInfo rowFilterInfo, List accesses, List users, List groups, List roles, List conditions, Boolean delegateAdmin) { + super(accesses, users, groups, roles, conditions, delegateAdmin); + + setRowFilterInfo(rowFilterInfo); + } + + /** + * @return the rowFilterInfo + */ + public RangerPolicyItemRowFilterInfo getRowFilterInfo() { + return rowFilterInfo; + } + + /** + * @param rowFilterInfo the rowFilterInfo to set + */ + public void setRowFilterInfo(RangerPolicyItemRowFilterInfo rowFilterInfo) { + this.rowFilterInfo = rowFilterInfo == null ? new RangerPolicyItemRowFilterInfo() : rowFilterInfo; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = super.hashCode(); + result = prime * result + ((rowFilterInfo == null) ? 0 : rowFilterInfo.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if(! super.equals(obj)) + return false; + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + RangerRowFilterPolicyItem other = (RangerRowFilterPolicyItem) obj; + if (rowFilterInfo == null) { + if (other.rowFilterInfo != null) + return false; + } else if (!rowFilterInfo.equals(other.rowFilterInfo)) + return false; + return true; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerRowFilterPolicyItem={"); + + super.toString(sb); + + sb.append("rowFilterInfo={"); + if(rowFilterInfo != null) { + rowFilterInfo.toString(sb); + } + sb.append("} "); + + sb.append("}"); + + return sb; + } + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class RangerPolicyItemAccess implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private String type; + private Boolean isAllowed; + + public RangerPolicyItemAccess() { + this(null, null); + } + + public RangerPolicyItemAccess(String type) { + this(type, null); + } + + public RangerPolicyItemAccess(String type, Boolean isAllowed) { + setType(type); + setIsAllowed(isAllowed); + } + + /** + * @return the type + */ + public String getType() { + return type; + } + + /** + * @param type the type to set + */ + public void setType(String type) { + this.type = type; + } + + /** + * @return the isAllowed + */ + public Boolean getIsAllowed() { + return isAllowed; + } + + /** + * @param isAllowed the isAllowed to set + */ + public void setIsAllowed(Boolean isAllowed) { + this.isAllowed = isAllowed == null ? Boolean.TRUE : isAllowed; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerPolicyItemAccess={"); + sb.append("type={").append(type).append("} "); + sb.append("isAllowed={").append(isAllowed).append("} "); + sb.append("}"); + + return sb; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + + ((isAllowed == null) ? 0 : isAllowed.hashCode()); + result = prime * result + ((type == null) ? 0 : type.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + RangerPolicyItemAccess other = (RangerPolicyItemAccess) obj; + if (isAllowed == null) { + if (other.isAllowed != null) + return false; + } else if (!isAllowed.equals(other.isAllowed)) + return false; + if (type == null) { + if (other.type != null) + return false; + } else if (!type.equals(other.type)) + return false; + return true; + } + + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class RangerPolicyItemCondition implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private String type; + private List values; + + public RangerPolicyItemCondition() { + this(null, null); + } + + public RangerPolicyItemCondition(String type, List values) { + setType(type); + setValues(values); + } + + /** + * @return the type + */ + public String getType() { + return type; + } + + /** + * @param type the type to set + */ + public void setType(String type) { + this.type = type; + } + + /** + * @return the value + */ + public List getValues() { + return values; + } + + /** + * @param values the value to set + */ + public void setValues(List values) { + if (this.values == null) { + this.values = new ArrayList<>(); + } + + if(this.values == values) { + return; + } + + this.values.clear(); + + if(values != null) { + this.values.addAll(values); + } + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerPolicyCondition={"); + sb.append("type={").append(type).append("} "); + sb.append("values={"); + if(values != null) { + for(String value : values) { + sb.append(value).append(" "); + } + } + sb.append("} "); + sb.append("}"); + + return sb; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = 1; + result = prime * result + ((type == null) ? 0 : type.hashCode()); + result = prime * result + + ((values == null) ? 0 : values.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + RangerPolicyItemCondition other = (RangerPolicyItemCondition) obj; + if (type == null) { + if (other.type != null) + return false; + } else if (!type.equals(other.type)) + return false; + if (values == null) { + if (other.values != null) + return false; + } else if (!values.equals(other.values)) + return false; + return true; + } + + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class RangerPolicyItemDataMaskInfo implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private String dataMaskType; + private String conditionExpr; + private String valueExpr; + + public RangerPolicyItemDataMaskInfo() { } + + public RangerPolicyItemDataMaskInfo(String dataMaskType, String conditionExpr, String valueExpr) { + setDataMaskType(dataMaskType); + setConditionExpr(conditionExpr); + setValueExpr(valueExpr); + } + + public RangerPolicyItemDataMaskInfo(RangerPolicyItemDataMaskInfo that) { + this.dataMaskType = that.dataMaskType; + this.conditionExpr = that.conditionExpr; + this.valueExpr = that.valueExpr; + } + + public String getDataMaskType() { + return dataMaskType; + } + + public void setDataMaskType(String dataMaskType) { + this.dataMaskType = dataMaskType; + } + + public String getConditionExpr() { + return conditionExpr; + } + + public void setConditionExpr(String conditionExpr) { + this.conditionExpr = conditionExpr; + } + + public String getValueExpr() { + return valueExpr; + } + + public void setValueExpr(String valueExpr) { + this.valueExpr = valueExpr; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = super.hashCode(); + result = prime * result + ((dataMaskType == null) ? 0 : dataMaskType.hashCode()); + result = prime * result + ((conditionExpr == null) ? 0 : conditionExpr.hashCode()); + result = prime * result + ((valueExpr == null) ? 0 : valueExpr.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + RangerPolicyItemDataMaskInfo other = (RangerPolicyItemDataMaskInfo) obj; + if (dataMaskType == null) { + if (other.dataMaskType != null) + return false; + } else if (!dataMaskType.equals(other.dataMaskType)) + return false; + if (conditionExpr == null) { + if (other.conditionExpr != null) + return false; + } else if (!conditionExpr.equals(other.conditionExpr)) + return false; + if (valueExpr == null) { + if (other.valueExpr != null) + return false; + } else if (!valueExpr.equals(other.valueExpr)) + return false; + return true; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerPolicyItemDataMaskInfo={"); + + sb.append("dataMaskType={").append(dataMaskType).append("} "); + sb.append("conditionExpr={").append(conditionExpr).append("} "); + sb.append("valueExpr={").append(valueExpr).append("} "); + + sb.append("}"); + + return sb; + } + } + + @JsonInclude(JsonInclude.Include.NON_NULL) + @XmlRootElement + @XmlAccessorType(XmlAccessType.FIELD) + public static class RangerPolicyItemRowFilterInfo implements java.io.Serializable { + private static final long serialVersionUID = 1L; + + private String filterExpr; + + public RangerPolicyItemRowFilterInfo() { } + + public RangerPolicyItemRowFilterInfo(String filterExpr) { + setFilterExpr(filterExpr); + } + + public RangerPolicyItemRowFilterInfo(RangerPolicyItemRowFilterInfo that) { + this.filterExpr = that.filterExpr; + } + + public String getFilterExpr() { + return filterExpr; + } + + public void setFilterExpr(String filterExpr) { + this.filterExpr = filterExpr; + } + + @Override + public int hashCode() { + final int prime = 31; + int result = super.hashCode(); + result = prime * result + ((filterExpr == null) ? 0 : filterExpr.hashCode()); + return result; + } + + @Override + public boolean equals(Object obj) { + if (this == obj) + return true; + if (obj == null) + return false; + if (getClass() != obj.getClass()) + return false; + RangerPolicyItemRowFilterInfo other = (RangerPolicyItemRowFilterInfo) obj; + if (filterExpr == null) { + if (other.filterExpr != null) + return false; + } else if (!filterExpr.equals(other.filterExpr)) + return false; + return true; + } + + @Override + public String toString( ) { + StringBuilder sb = new StringBuilder(); + + toString(sb); + + return sb.toString(); + } + + public StringBuilder toString(StringBuilder sb) { + sb.append("RangerPolicyItemRowFilterInfo={"); + + sb.append("filterExpr={").append(filterExpr).append("} "); + + sb.append("}"); + + return sb; + } + } +} diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java index adfe24af1d..def6c6c0dd 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/model/RangerPolicyDelta.java @@ -73,6 +73,9 @@ public RangerPolicyDelta(final Long id, final Integer changeType, final Long pol @JsonIgnore public Long getPolicyId() { return policy != null ? policy.getId() : null; } + @JsonIgnore + public String getPolicyGuid() { return policy != null ? policy.getGuid() : null; } + @JsonIgnore public String getZoneName() { return policy != null ? policy.getZoneName() : null; } @@ -94,6 +97,7 @@ public String toString() { + ", serviceType:" + getServiceType() + ", policyType:" + getPolicyType() + ", policyId:[" + getPolicyId() + "]" + + ", policyGuid:[" + getPolicyGuid() + "]" + ", policy:[" + policy +"]"; } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyRepository.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyRepository.java index 5b3b469fb0..4324ccb2cd 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyRepository.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/policyengine/RangerPolicyRepository.java @@ -90,7 +90,7 @@ enum AuditModeEnum { private List dataMaskPolicyEvaluators; private List rowFilterPolicyEvaluators; private final List auditPolicyEvaluators; - private Map policyEvaluatorsMap; + private Map policyEvaluatorsMap; private boolean isContextEnrichersShared = false; private boolean isPreCleaned = false; @@ -654,9 +654,9 @@ public List getLikelyMatchPolicyEvaluators(RangerAccessRe } - public Map getPolicyEvaluatorsMap() { return policyEvaluatorsMap; } + public Map getPolicyEvaluatorsMap() { return policyEvaluatorsMap; } - RangerPolicyEvaluator getPolicyEvaluator(Long id) { + RangerPolicyEvaluator getPolicyEvaluator(String id) { return policyEvaluatorsMap.get(id); } @@ -1252,17 +1252,17 @@ private void removeEvaluatorFromTrie(RangerPolicyEvaluator oldEvaluator, RangerR } } - private Map createPolicyEvaluatorsMap() { - Map tmpPolicyEvaluatorMap = new HashMap<>(); + private Map createPolicyEvaluatorsMap() { + Map tmpPolicyEvaluatorMap = new HashMap<>(); for (RangerPolicyEvaluator evaluator : getPolicyEvaluators()) { - tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator); + tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getGuid(), evaluator); } for (RangerPolicyEvaluator evaluator : getDataMaskPolicyEvaluators()) { - tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator); + tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getGuid(), evaluator); } for (RangerPolicyEvaluator evaluator : getRowFilterPolicyEvaluators()) { - tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getId(), evaluator); + tmpPolicyEvaluatorMap.put(evaluator.getPolicy().getGuid(), evaluator); } return tmpPolicyEvaluatorMap; @@ -1294,7 +1294,7 @@ private RangerPolicyEvaluator addPolicy(RangerPolicy policy) { } if (!RangerPolicy.POLICY_TYPE_AUDIT.equals(policy.getPolicyType())) { - policyEvaluatorsMap.put(policy.getId(), ret); + policyEvaluatorsMap.put(policy.getGuid(), ret); } } } @@ -1306,22 +1306,22 @@ private RangerPolicyEvaluator addPolicy(RangerPolicy policy) { return ret; } - private void removePolicy(Long id) { + private void removePolicy(String guid) { if (LOG.isDebugEnabled()) { - LOG.debug("==> RangerPolicyRepository.removePolicy(" + id +")"); + LOG.debug("==> RangerPolicyRepository.removePolicy(" + guid +")"); } Iterator iterator = policies.iterator(); while (iterator.hasNext()) { - if (id.equals(iterator.next().getId())) { + if (guid.equals(iterator.next().getGuid())) { iterator.remove(); //break; } } - policyEvaluatorsMap.remove(id); + policyEvaluatorsMap.remove(guid); if (LOG.isDebugEnabled()) { - LOG.debug("<== RangerPolicyRepository.removePolicy(" + id +")"); + LOG.debug("<== RangerPolicyRepository.removePolicy(" + guid +")"); } } @@ -1355,13 +1355,13 @@ private void deletePolicyEvaluator(RangerPolicyEvaluator evaluator) { } private RangerPolicyEvaluator update(final RangerPolicyDelta delta, final RangerPolicyEvaluator currentEvaluator) { - + LOG.info("PolicyDelta: RangerPolicyRepository.update is called, policyGuid: "+delta.getPolicyGuid()); if (LOG.isDebugEnabled()) { LOG.debug("==> RangerPolicyRepository.update(delta=" + delta + ", currentEvaluator=" + (currentEvaluator == null ? null : currentEvaluator.getPolicy()) + ")"); } Integer changeType = delta.getChangeType(); String policyType = delta.getPolicyType(); - Long policyId = delta.getPolicyId(); + String policyId = delta.getPolicyGuid(); RangerPolicy policy = delta.getPolicy(); @@ -1472,7 +1472,7 @@ private void updateResourceTrie(List deltas) { for (RangerPolicyDelta delta : deltas) { final Integer changeType = delta.getChangeType(); final String serviceType = delta.getServiceType(); - final Long policyId = delta.getPolicyId(); + final String policyId = delta.getPolicyGuid(); final String policyType = delta.getPolicyType(); if (!serviceType.equals(this.serviceDef.getName())) { diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java index 9cbbe74b7d..566fa862ec 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/service/RangerBasePlugin.java @@ -391,6 +391,7 @@ public void setPolicies(ServicePolicies policies) { RangerPolicyEngineImpl oldPolicyEngineImpl = (RangerPolicyEngineImpl) oldPolicyEngine; newPolicyEngine = RangerPolicyEngineImpl.getPolicyEngine(oldPolicyEngineImpl, policies); + //TODO: this looks like a mistake, second arg should be servicePolicies which has the applied delta } if (newPolicyEngine != null) { diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java index 457aa02f0e..b18906dde4 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/PolicyRefresher.java @@ -21,8 +21,12 @@ import com.google.gson.Gson; import com.google.gson.GsonBuilder; +import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasException; import org.apache.atlas.authz.admin.client.AtlasAuthAdminClient; import org.apache.atlas.policytransformer.CachePolicyTransformerImpl; +import org.apache.atlas.repository.audit.ESBasedAuditRepository; +import org.apache.commons.configuration.Configuration; import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -40,6 +44,8 @@ import java.util.concurrent.BlockingQueue; import java.util.concurrent.LinkedBlockingQueue; +import static org.apache.atlas.ApplicationProperties.DELTA_BASED_REFRESH; + public class PolicyRefresher extends Thread { private static final Log LOG = LogFactory.getLog(PolicyRefresher.class); @@ -64,6 +70,9 @@ public class PolicyRefresher extends Thread { private long lastActivationTimeInMillis; private boolean policiesSetInPlugin; private boolean serviceDefSetInPlugin; + private Configuration atlasConfig; + private boolean enableDeltaBasedRefresh; + private ESBasedAuditRepository auditRepository; public PolicyRefresher(RangerBasePlugin plugIn) { @@ -104,6 +113,16 @@ public PolicyRefresher(RangerBasePlugin plugIn) { this.userStoreProvider = new RangerUserStoreProvider(getServiceType(), appId, getServiceName(), atlasAuthAdminClient, cacheDir, pluginConfig); this.pollingIntervalMs = pluginConfig.getLong(propertyPrefix + ".policy.pollIntervalMs", 30 * 1000); + try { + this.atlasConfig = ApplicationProperties.get(); + this.auditRepository = new ESBasedAuditRepository(atlasConfig); + this.auditRepository.start(); + this.enableDeltaBasedRefresh = this.atlasConfig.getBoolean(DELTA_BASED_REFRESH, false); + } catch (AtlasException e) { + LOG.error("PolicyDelta: Error while reading atlas configuration", e); + this.enableDeltaBasedRefresh = false; + } + setName("PolicyRefresher(serviceName=" + serviceName + ")-" + getId()); if(LOG.isDebugEnabled()) { @@ -316,13 +335,20 @@ private ServicePolicies loadPolicyfromPolicyAdmin() throws RangerServiceNotFound try { - if (serviceName.equals("atlas") && plugIn.getTypeRegistry() != null && lastUpdatedTiemInMillis == -1) { + if (serviceName.equals("atlas") && plugIn.getTypeRegistry() != null) { RangerRESTUtils restUtils = new RangerRESTUtils(); - CachePolicyTransformerImpl transformer = new CachePolicyTransformerImpl(plugIn.getTypeRegistry()); - - svcPolicies = transformer.getPolicies(serviceName, - restUtils.getPluginId(serviceName, plugIn.getAppId()), - lastUpdatedTiemInMillis, null); + CachePolicyTransformerImpl transformer = new CachePolicyTransformerImpl(plugIn.getTypeRegistry(), auditRepository); + if (lastUpdatedTiemInMillis == -1) { + svcPolicies = transformer.getPoliciesAll(serviceName, + restUtils.getPluginId(serviceName, plugIn.getAppId()), + lastUpdatedTiemInMillis, null); + } else if (this.enableDeltaBasedRefresh) { + svcPolicies = transformer.getPoliciesDelta(serviceName, + restUtils.getPluginId(serviceName, plugIn.getAppId()), + lastUpdatedTiemInMillis); + } else { + svcPolicies = atlasAuthAdminClient.getServicePoliciesIfUpdated(lastUpdatedTiemInMillis); + } } else { svcPolicies = atlasAuthAdminClient.getServicePoliciesIfUpdated(lastUpdatedTiemInMillis); } diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java index e3cb0f3697..b92ae004d4 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/RangerPolicyDeltaUtil.java @@ -79,7 +79,7 @@ public static List applyDeltas(List policies, List applyDeltas(List policies, List applyDeltas(List policies, List deltas, String compo for (RangerPolicyDelta delta : deltas) { final Integer changeType = delta.getChangeType(); - final Long policyId = delta.getPolicyId(); + final String policyGuid = delta.getPolicyGuid(); if (changeType == null) { isValid = false; @@ -171,7 +167,7 @@ public static boolean isValidDeltas(List deltas, String compo && changeType != RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE && changeType != RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE) { isValid = false; - } else if (policyId == null) { + } else if (policyGuid == null) { isValid = false; } else { final String serviceType = delta.getServiceType(); diff --git a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java index 9713e85034..0529914e48 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/plugin/util/ServicePolicies.java @@ -392,7 +392,7 @@ static public TagPolicies copyHeader(TagPolicies source, String componentService return ret; } - public static ServicePolicies applyDelta(final ServicePolicies servicePolicies, RangerPolicyEngineImpl policyEngine) { + public static ServicePolicies applyDelta(final ServicePolicies servicePolicies, RangerPolicyEngineImpl policyEngine) { ServicePolicies ret = copyHeader(servicePolicies); List oldResourcePolicies = policyEngine.getResourcePolicies(); diff --git a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java index 89c3b6e5cd..f092a488a6 100644 --- a/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java +++ b/auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java @@ -23,9 +23,14 @@ import org.apache.atlas.RequestContext; import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.audit.AuditSearchParams; +import org.apache.atlas.model.audit.EntityAuditEventV2; +import org.apache.atlas.model.audit.EntityAuditEventV2.EntityAuditActionV2; +import org.apache.atlas.model.audit.EntityAuditSearchResult; import org.apache.atlas.model.discovery.AtlasSearchResult; import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.plugin.model.RangerPolicyDelta; import org.apache.atlas.plugin.util.ServicePolicies; import org.apache.atlas.plugin.model.RangerPolicy; import org.apache.atlas.plugin.model.RangerPolicy.RangerDataMaskPolicyItem; @@ -37,6 +42,7 @@ import org.apache.atlas.plugin.model.RangerServiceDef; import org.apache.atlas.plugin.model.RangerValiditySchedule; import org.apache.atlas.plugin.util.ServicePolicies.TagPolicies; +import org.apache.atlas.repository.audit.ESBasedAuditRepository; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.janus.AtlasJanusGraph; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; @@ -57,9 +63,7 @@ import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; -import static org.apache.atlas.repository.Constants.NAME; -import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; -import static org.apache.atlas.repository.Constants.SERVICE_ENTITY_TYPE; +import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_CATEGORY; import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_CONNECTION_QN; import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_IS_ENABLED; @@ -109,11 +113,15 @@ public class CachePolicyTransformerImpl { private PurposeCachePolicyTransformer purposeTransformer; private AtlasEntityHeader service; + private final ESBasedAuditRepository auditRepository; + + private final Map auditEventToDeltaChangeType; @Inject - public CachePolicyTransformerImpl(AtlasTypeRegistry typeRegistry) throws AtlasBaseException { + public CachePolicyTransformerImpl(AtlasTypeRegistry typeRegistry, ESBasedAuditRepository auditRepository) throws AtlasBaseException { this.graph = new AtlasJanusGraph(); this.entityRetriever = new EntityGraphRetriever(graph, typeRegistry); + this.auditRepository = auditRepository; personaTransformer = new PersonaCachePolicyTransformer(entityRetriever); purposeTransformer = new PurposeCachePolicyTransformer(entityRetriever); @@ -124,13 +132,81 @@ public CachePolicyTransformerImpl(AtlasTypeRegistry typeRegistry) throws AtlasBa LOG.error("Failed to initialize discoveryService"); throw new AtlasBaseException(e.getCause()); } + + this.auditEventToDeltaChangeType = new HashMap<>(); + this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_CREATE, RangerPolicyDelta.CHANGE_TYPE_POLICY_CREATE); + this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_UPDATE, RangerPolicyDelta.CHANGE_TYPE_POLICY_UPDATE); + this.auditEventToDeltaChangeType.put(EntityAuditActionV2.ENTITY_DELETE, RangerPolicyDelta.CHANGE_TYPE_POLICY_DELETE); } public AtlasEntityHeader getService() { return service; } - public ServicePolicies getPolicies(String serviceName, String pluginId, Long lastUpdatedTime, Date latestEditTime) { + public ServicePolicies getPoliciesDelta(String serviceName, String pluginId, Long lastUpdatedTime) { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getPoliciesDelta." + serviceName); + + ServicePolicies servicePolicies = new ServicePolicies(); + + try { + servicePolicies.setServiceName(serviceName); + + service = getServiceEntity(serviceName); + servicePolicies.setPolicyVersion(-1L); + servicePolicies.setPolicyUpdateTime(new Date()); + + if (service != null) { + servicePolicies.setServiceName(serviceName); + servicePolicies.setServiceId(service.getGuid()); + + String serviceDefName = String.format(RESOURCE_SERVICE_DEF_PATTERN, serviceName); + servicePolicies.setServiceDef(getResourceAsObject(serviceDefName, RangerServiceDef.class)); + + List policiesDelta = getServicePoliciesDelta(service, 250, lastUpdatedTime); + servicePolicies.setPolicyDeltas(policiesDelta); + + + //Process tag based policies + String tagServiceName = (String) service.getAttribute(ATTR_SERVICE_TAG_SERVICE); + if (StringUtils.isNotEmpty(tagServiceName)) { + AtlasEntityHeader tagService = getServiceEntity(tagServiceName); + + if (tagService != null) { + List tagRangerPolicies = getServicePolicies(tagService, 0, null); + + TagPolicies tagPolicies = new TagPolicies(); + + tagPolicies.setServiceName(tagServiceName); + tagPolicies.setPolicyUpdateTime(new Date()); + tagPolicies.setServiceId(tagService.getGuid()); + tagPolicies.setPolicyVersion(-1L); + + String tagServiceDefName = String.format(RESOURCE_SERVICE_DEF_PATTERN, tagService.getAttribute(NAME)); + tagPolicies.setServiceDef(getResourceAsObject(tagServiceDefName, RangerServiceDef.class)); + + servicePolicies.setTagPolicies(tagPolicies); + servicePolicies.getTagPolicies().setPolicies(tagRangerPolicies); + LOG.info("PolicyDelta: {}: Found tag policies - {}", serviceName, tagRangerPolicies.size()); + } + } + + + + LOG.info("PolicyDelta: {}: Found {} policies", serviceName, policiesDelta.size()); + LOG.info("PolicyDelta: Found and set {} policies as delta and {} tag policies", servicePolicies.getPolicyDeltas().size(), servicePolicies.getTagPolicies().getPolicies().size()); + } + + } catch (Exception e) { + LOG.error("PolicyDelta: {}: ERROR in getPoliciesDelta {}: {}", serviceName, e.getMessage(), e); + return null; + } + + RequestContext.get().endMetricRecord(recorder); + return servicePolicies; + } + + + public ServicePolicies getPoliciesAll(String serviceName, String pluginId, Long lastUpdatedTime, Date latestEditTime) { //TODO: return only if updated AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getPolicies." + serviceName); @@ -208,7 +284,7 @@ private List getServicePolicies(AtlasEntityHeader service, int bat int sleepFor = 500; for (int attempt = 0; attempt <= maxAttempts; attempt++) { try { - atlasPolicies = getAtlasPolicies(serviceName, batchSize, latestEditTime); + atlasPolicies = getAtlasPolicies(serviceName, batchSize, latestEditTime, new ArrayList<>()); break; } catch (AtlasBaseException e) { LOG.error("ES_SYNC_FIX: {}: ERROR in getServicePolicies: {}", serviceName, e.getMessage()); @@ -227,6 +303,83 @@ private List getServicePolicies(AtlasEntityHeader service, int bat return servicePolicies; } + private List getServicePoliciesDelta(AtlasEntityHeader service, int batchSize, Long lastUpdatedTime) throws AtlasBaseException, IOException { + + String serviceName = (String) service.getAttribute("name"); + String serviceType = (String) service.getAttribute("authServiceType"); + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.getServicePoliciesWithDelta." + serviceName); + + List policyDeltas = new ArrayList<>(); + + // TODO: when getServicePolicies (without delta) is removed, merge the pagination for audit logs and policy fetch into one + List auditEvents = queryPoliciesAuditLogs(serviceName, lastUpdatedTime, batchSize); + Map policiesWithChangeType = new HashMap<>(); + for (EntityAuditEventV2 event : auditEvents) { + if (POLICY_ENTITY_TYPE.equals(event.getTypeName()) && !policiesWithChangeType.containsKey(event.getEntityId())) { + policiesWithChangeType.put(event.getEntityId(), event.getAction()); + } + } + LOG.info("PolicyDelta: {}: Total audit logs found = {}, events for {} ({}) = {}", serviceName, auditEvents.size(), POLICY_ENTITY_TYPE, policiesWithChangeType.size(), policiesWithChangeType); + if (policiesWithChangeType.isEmpty()) { + return policyDeltas; + } + + ArrayList policyGuids = new ArrayList<>(policiesWithChangeType.keySet()); + List atlasPolicies = getAtlasPolicies(serviceName, batchSize, null, policyGuids); + + List rangerPolicies = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(atlasPolicies)) { + rangerPolicies = transformAtlasPoliciesToRangerPolicies(atlasPolicies, serviceType, serviceName); + } + + for (RangerPolicy policy : rangerPolicies) { + Integer changeType = auditEventToDeltaChangeType.get(policiesWithChangeType.get(policy.getGuid())); + RangerPolicyDelta delta = new RangerPolicyDelta(policy.getId(), changeType, policy.getVersion(), policy); + policyDeltas.add(delta); + } + LOG.info("PolicyDelta: {}: atlas policies found = {}, delta created = {}", serviceName, atlasPolicies.size(), policyDeltas.size()); + RequestContext.get().endMetricRecord(recorder); + + return policyDeltas; + } + + private List queryPoliciesAuditLogs(String serviceName, Long afterTime, int batchSize) { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl.queryPoliciesAuditLogs." + serviceName); + + List entityUpdateToWatch = new ArrayList<>(); + entityUpdateToWatch.add(POLICY_ENTITY_TYPE); + entityUpdateToWatch.add(PURPOSE_ENTITY_TYPE); + + AuditSearchParams parameters = new AuditSearchParams(); + Map dsl = getMap("size", batchSize); + + List> mustClauseList = new ArrayList<>(); + mustClauseList.add(getMap("terms", getMap("typeName", entityUpdateToWatch))); + afterTime = afterTime == -1 ? 0 : afterTime; + mustClauseList.add(getMap("range", getMap("created", getMap("gte", afterTime)))); + + List> sortList = new ArrayList<>(0); + sortList.add(getMap("created", getMap("order", "desc"))); + dsl.put("sort", sortList); + + dsl.put("query", getMap("bool", getMap("must", mustClauseList))); + + parameters.setDsl(dsl); + + List events = new ArrayList<>(); + try { + EntityAuditSearchResult result = auditRepository.searchEvents(parameters.getQueryString()); + if (result != null && !CollectionUtils.isEmpty(result.getEntityAudits())) { + events = result.getEntityAudits(); + } + } catch (AtlasBaseException e) { + LOG.error("ERROR in queryPoliciesAuditLogs while fetching entity audits {}: ", e.getMessage(), e); + } finally { + RequestContext.get().endMetricRecord(recorder); + } + return events; + } + private List transformAtlasPoliciesToRangerPolicies(List atlasPolicies, String serviceType, String serviceName) throws IOException, AtlasBaseException { @@ -461,7 +614,7 @@ private List getPolicyValiditySchedule(AtlasEntityHeader return ret; } - private List getAtlasPolicies(String serviceName, int batchSize, Date latestEditTime) throws AtlasBaseException { + private List getAtlasPolicies(String serviceName, int batchSize, Date latestEditTime, List policyGuids) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("CachePolicyTransformerImpl."+service+".getAtlasPolicies"); List ret = new ArrayList<>(); @@ -493,6 +646,10 @@ private List getAtlasPolicies(String serviceName, int batchSi mustClauseList.add(getMap("term", getMap(ATTR_POLICY_IS_ENABLED, true))); mustClauseList.add(getMap("match", getMap("__state", Id.EntityState.ACTIVE))); + if (!policyGuids.isEmpty()) { + mustClauseList.add(getMap("terms", getMap("__guid", policyGuids))); + } + dsl.put("query", getMap("bool", getMap("must", mustClauseList))); List sortList = new ArrayList<>(0); diff --git a/intg/src/main/java/org/apache/atlas/ApplicationProperties.java b/intg/src/main/java/org/apache/atlas/ApplicationProperties.java index 78b487a8dd..dc198ba2fa 100644 --- a/intg/src/main/java/org/apache/atlas/ApplicationProperties.java +++ b/intg/src/main/java/org/apache/atlas/ApplicationProperties.java @@ -70,6 +70,7 @@ public final class ApplicationProperties extends PropertiesConfiguration { public static final boolean DEFAULT_INDEX_RECOVERY = true; public static final AtlasRunMode DEFAULT_ATLAS_RUN_MODE = AtlasRunMode.PROD; public static final String INDEX_SEARCH_MAX_RESULT_SET_SIZE = "atlas.graph.index.search.max-result-set-size"; + public static final String DELTA_BASED_REFRESH = "atlas.authorizer.enable.delta_based_refresh"; public static final SimpleEntry DB_CACHE_CONF = new SimpleEntry<>("atlas.graph.cache.db-cache", "true"); public static final SimpleEntry DB_CACHE_CLEAN_WAIT_CONF = new SimpleEntry<>("atlas.graph.cache.db-cache-clean-wait", "20"); diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java index 69f1b29b98..a8a535cc7c 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AuthREST.java @@ -155,7 +155,7 @@ public ServicePolicies downloadPolicies(@PathParam("serviceName") final String s return null; } - ServicePolicies ret = policyTransformer.getPolicies(serviceName, pluginId, lastUpdatedTime, new Date(latestEditTime)); + ServicePolicies ret = policyTransformer.getPoliciesAll(serviceName, pluginId, lastUpdatedTime, new Date(latestEditTime)); updateLastSync(serviceName);