Merge branch 'master' into feat/lineage-add-node-depth

.github/workflows/chart-release-dispatcher.yaml

@@ -6,6 +6,7 @@ on:
       - "Java CI with Maven"
     branches:
       - master
+      - staging
       - beta
     types:
       - completed

addons/policies/bootstrap_entity_policies.json

@@ -2991,6 +2991,7 @@
                     "$api-token-default-access"
                 ],
                 "policyResourceCategory": "ENTITY",
+                "isPolicyEnabled": false,
                 "policyResources":
                 [
                     "entity-type:DataProduct",

auth-agents-common/src/main/java/org/apache/atlas/policytransformer/CachePolicyTransformerImpl.java

@@ -53,6 +53,7 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.ArrayList;
+import java.util.Arrays;
 import java.util.Collections;
 import java.util.Date;
 import java.util.HashMap;
@@ -71,6 +72,7 @@
 import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_PRIORITY;
 import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_SERVICE_NAME;
 import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_SUB_CATEGORY;
+import static org.apache.atlas.repository.util.AccessControlUtils.POLICY_CATEGORY_DATAMESH;
 import static org.apache.atlas.repository.util.AccessControlUtils.POLICY_CATEGORY_PERSONA;
 import static org.apache.atlas.repository.util.AccessControlUtils.POLICY_CATEGORY_PURPOSE;
 import static org.apache.atlas.repository.util.AccessControlUtils.getIsPolicyEnabled;
@@ -239,6 +241,18 @@ private List<RangerPolicy> transformAtlasPoliciesToRangerPolicies(List<AtlasEnti
                         rangerPolicies.add(toRangerPolicy(transformedPolicy, serviceType));
                     }
 
+                }
+                else if (POLICY_CATEGORY_DATAMESH.equals(policyCategory)) {
+                    RangerPolicy rangerPolicy = getRangerPolicy(atlasPolicy, serviceType);
+
+                    //GET policy Item
+                    setPolicyItems(rangerPolicy, atlasPolicy);
+
+                    //GET policy Resources
+                    setPolicyResourcesForDatameshPolicies(rangerPolicy, atlasPolicy);
+
+                    rangerPolicies.add(rangerPolicy);
+
                 } else {
                     rangerPolicies.add(toRangerPolicy(atlasPolicy, serviceType));
                 }
@@ -264,6 +278,26 @@ private RangerPolicy toRangerPolicy(AtlasEntityHeader atlasPolicy, String servic
     }
 
     private void setPolicyResources(RangerPolicy rangerPolicy, AtlasEntityHeader atlasPolicy) throws IOException {
+        rangerPolicy.setResources(getFinalResources(atlasPolicy));
+    }
+
+    private void setPolicyResourcesForDatameshPolicies(RangerPolicy rangerPolicy, AtlasEntityHeader atlasPolicy) {
+        Map<String, RangerPolicyResource> resources = getFinalResources(atlasPolicy);
+
+        if (!resources.containsKey("entity-classification")) {
+            RangerPolicyResource resource = new RangerPolicyResource(Arrays.asList("*"), false, false);
+            resources.put("entity-classification", resource);
+        }
+
+        if (!resources.containsKey("entity-type")) {
+            RangerPolicyResource resource = new RangerPolicyResource(Arrays.asList("*"), false, false);
+            resources.put("entity-type", resource);
+        }
+
+        rangerPolicy.setResources(resources);
+    }
+
+    private Map<String, RangerPolicyResource> getFinalResources(AtlasEntityHeader atlasPolicy) {
         List<String> atlasResources = (List<String>) atlasPolicy.getAttribute("policyResources");
 
         Map<String, List<String>> resourceValuesMap = new HashMap<>();
@@ -285,7 +319,7 @@ private void setPolicyResources(RangerPolicy rangerPolicy, AtlasEntityHeader atl
             resources.put(key, resource);
         }
 
-        rangerPolicy.setResources(resources);
+        return resources;
     }
 
     private <T> T getResourceAsObject(String resourceName, Class<T> clazz) throws IOException {

common/src/main/java/org/apache/atlas/repository/Constants.java

@@ -25,7 +25,6 @@
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
-import java.io.File;
 import java.io.IOException;
 import java.nio.charset.StandardCharsets;
 import java.nio.file.Files;
@@ -259,6 +258,8 @@ public final class Constants {
     public static final String CLASSIFICATION_VERTEX_PROPAGATE_KEY            = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "propagate");
     public static final String CLASSIFICATION_VERTEX_REMOVE_PROPAGATIONS_KEY  = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "removePropagations");
     public static final String CLASSIFICATION_VERTEX_RESTRICT_PROPAGATE_THROUGH_LINEAGE= encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "restrictPropagationThroughLineage");
+
+    public static final String CLASSIFICATION_VERTEX_RESTRICT_PROPAGATE_THROUGH_HIERARCHY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "restrictPropagationThroughHierarchy");
     public static final String CLASSIFICATION_VERTEX_NAME_KEY                 = encodePropertyKey(TYPE_NAME_PROPERTY_KEY);
     public static final String CLASSIFICATION_EDGE_NAME_PROPERTY_KEY          = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "name");
     public static final String CLASSIFICATION_EDGE_IS_PROPAGATED_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "isPropagated");
@@ -374,13 +375,21 @@ public enum SupportedFileExtensions { XLSX, XLS, CSV }
     public static final String CLASSIFICATION_PROPAGATION_MODE_DEFAULT  ="DEFAULT";
     public static final String CLASSIFICATION_PROPAGATION_MODE_RESTRICT_LINEAGE  ="RESTRICT_LINEAGE";
 
-    public static final HashMap<String, ArrayList<String>> CLASSIFICATION_PROPAGATION_EXCLUSION_MAP = new HashMap<String, ArrayList<String>>(){{
+    public static final String CLASSIFICATION_PROPAGATION_MODE_RESTRICT_HIERARCHY  ="RESTRICT_HIERARCHY";
+
+
+    public static final HashMap<String, ArrayList<String>> CLASSIFICATION_PROPAGATION_MODE_LABELS_MAP = new HashMap<String, ArrayList<String>>(){{
         put(CLASSIFICATION_PROPAGATION_MODE_RESTRICT_LINEAGE, new ArrayList<>(
                 Arrays.asList(CATALOG_PROCESS_INPUT_RELATIONSHIP_LABEL,
                 CATALOG_PROCESS_OUTPUT_RELATIONSHIP_LABEL,
                 COLUMN_LINEAGE_RELATIONSHIP_LABEL
         )));
         put(CLASSIFICATION_PROPAGATION_MODE_DEFAULT, null);
+        put(CLASSIFICATION_PROPAGATION_MODE_RESTRICT_HIERARCHY, new ArrayList<>(
+                Arrays.asList(CATALOG_PROCESS_INPUT_RELATIONSHIP_LABEL,
+                        CATALOG_PROCESS_OUTPUT_RELATIONSHIP_LABEL,
+                        COLUMN_LINEAGE_RELATIONSHIP_LABEL
+                )));
     }};
 
     public static final String ATTR_ADMIN_USERS = "adminUsers";

intg/src/main/java/org/apache/atlas/AtlasConfiguration.java

@@ -111,7 +111,9 @@ public enum AtlasConfiguration {
     HERACLES_CLIENT_PAGINATION_SIZE("atlas.heracles.admin.resource-pagination-size", 100),
     HERACLES_API_SERVER_URL("atlas.heracles.api.service.url", "http://heracles-service.heracles.svc.cluster.local"),
 
-    INDEXSEARCH_ASYNC_SEARCH_KEEP_ALIVE_TIME_IN_SECONDS("atlas.indexsearch.async.search.keep.alive.time.in.seconds", 300);
+    INDEXSEARCH_ASYNC_SEARCH_KEEP_ALIVE_TIME_IN_SECONDS("atlas.indexsearch.async.search.keep.alive.time.in.seconds", 300),
+
+    ATLAS_MAINTENANCE_MODE("atlas.maintenance.mode", false);
 
 
     private static final Configuration APPLICATION_PROPERTIES;

intg/src/main/java/org/apache/atlas/AtlasErrorCode.java

@@ -274,6 +274,8 @@ public enum AtlasErrorCode {
     RUNTIME_EXCEPTION(500, "ATLAS-500-00-020", "Runtime exception {0}"),
     KEYCLOAK_INIT_FAILED(500, "ATLAS-500-00-022", "Failed to initialize keycloak client: {0}"),
 
+    MAINTENANCE_MODE_ENABLED(503, "ATLAS-503-00-001", "Atlas is in maintenance mode for this specific operation. Please try again later."),
+
     BATCH_SIZE_TOO_LARGE(406, "ATLAS-406-00-001", "Batch size is too large, please use a smaller batch size"),
 
 

intg/src/main/java/org/apache/atlas/model/discovery/IndexSearchParams.java

@@ -27,6 +27,7 @@ public class IndexSearchParams extends SearchParams {
     * (this will include related attributes which has relationshipStatus as DELETED along with ACTIVE ones)
     * */
     private boolean allowDeletedRelations;
+    private boolean accessControlExclusive;
 
     @Override
     public String getQuery() {
@@ -46,6 +47,14 @@ public boolean isAllowDeletedRelations() {
         return allowDeletedRelations;
     }
 
+    public boolean isAccessControlExclusive() {
+        return accessControlExclusive;
+    }
+
+    public void setAccessControlExclusive(boolean accessControlExclusive) {
+        this.accessControlExclusive = accessControlExclusive;
+    }
+
     public void setAllowDeletedRelations(boolean allowDeletedRelations) {
         this.allowDeletedRelations = allowDeletedRelations;
     }
@@ -78,6 +87,7 @@ public String toString() {
                 ", persona='" + persona + '\'' +
                 ", queryString='" + queryString + '\'' +
                 ", allowDeletedRelations=" + allowDeletedRelations +
+                ", accessControlExclusive=" + accessControlExclusive +
                 ", utmTags="+ getUtmTags() +
                 '}';
     }

intg/src/main/java/org/apache/atlas/model/instance/AtlasClassification.java

@@ -62,6 +62,8 @@ public class AtlasClassification extends AtlasStruct implements Serializable {
     private Boolean            removePropagationsOnEntityDelete  = null;
     private Boolean            restrictPropagationThroughLineage = null;
 
+    private Boolean            restrictPropagationThroughHierarchy = null;
+
     public AtlasClassification() {
         this(null, null);
     }
@@ -93,9 +95,18 @@ public AtlasClassification(AtlasClassification other) {
             setDisplayName(other.getDisplayName());
             setRemovePropagationsOnEntityDelete(other.getRemovePropagationsOnEntityDelete());
             setRestrictPropagationThroughLineage(other.getRestrictPropagationThroughLineage());
+            setRestrictPropagationThroughHierarchy(other.getRestrictPropagationThroughHierarchy());
         }
     }
 
+    public void setRestrictPropagationThroughHierarchy(Boolean restrictPropagationThroughHierarchy) {
+        this.restrictPropagationThroughHierarchy = restrictPropagationThroughHierarchy;
+    }
+
+    public Boolean getRestrictPropagationThroughHierarchy() {
+        return this.restrictPropagationThroughHierarchy;
+    }
+
     public String getDisplayName() {
         return displayName;
     }
@@ -178,7 +189,8 @@ public boolean equals(Object o) {
                Objects.equals(removePropagationsOnEntityDelete, that.removePropagationsOnEntityDelete) &&
                Objects.equals(entityGuid, that.entityGuid) &&
                entityStatus == that.entityStatus &&
-               Objects.equals(validityPeriods, that.validityPeriods) && Objects.equals(restrictPropagationThroughLineage, that.restrictPropagationThroughLineage);
+               Objects.equals(validityPeriods, that.validityPeriods) && Objects.equals(restrictPropagationThroughLineage, that.restrictPropagationThroughLineage) &&
+                Objects.equals(restrictPropagationThroughHierarchy, that.restrictPropagationThroughHierarchy);
     }
 
     public boolean checkForUpdate(Object o) {
@@ -191,12 +203,13 @@ public boolean checkForUpdate(Object o) {
                 Objects.equals(validityPeriods, that.validityPeriods) &&
                 (Objects.equals(propagate, that.propagate) || (propagate == null)) &&
                 (Objects.equals(removePropagationsOnEntityDelete, that.removePropagationsOnEntityDelete) || (removePropagationsOnEntityDelete == null)) &&
-                (Objects.equals(restrictPropagationThroughLineage, that.restrictPropagationThroughLineage) || (restrictPropagationThroughLineage == null));
+                (Objects.equals(restrictPropagationThroughLineage, that.restrictPropagationThroughLineage) || (restrictPropagationThroughLineage == null)) &&
+                (Objects.equals(restrictPropagationThroughHierarchy, that.restrictPropagationThroughHierarchy) || (restrictPropagationThroughHierarchy == null));
     }
 
     @Override
     public int hashCode() {
-        return Objects.hash(super.hashCode(), entityGuid, entityStatus, propagate, removePropagationsOnEntityDelete, restrictPropagationThroughLineage);
+        return Objects.hash(super.hashCode(), entityGuid, entityStatus, propagate, removePropagationsOnEntityDelete, restrictPropagationThroughLineage,restrictPropagationThroughHierarchy);
     }
 
     @Override
@@ -210,6 +223,7 @@ public String toString() {
         sb.append(", displayName=").append(displayName);
         sb.append(", validityPeriods=").append(validityPeriods);
         sb.append(", restrictPropagationThroughLineage=").append(restrictPropagationThroughLineage);
+        sb.append(", restrictPropagationThroughHierarchy=").append(restrictPropagationThroughHierarchy);
         sb.append('}');
         return sb.toString();
     }

repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java

@@ -1149,9 +1149,57 @@ private String getIndexName(IndexSearchParams params) throws AtlasBaseException
         String aliasName = parts[parts.length - 1];
 
         if (StringUtils.isNotEmpty(aliasName)) {
+            if(params.isAccessControlExclusive()) {
+                accessControlExclusiveDsl(params, aliasName);
+                aliasName = aliasName+","+VERTEX_INDEX_NAME;
+            }
             return aliasName;
         } else {
             throw new AtlasBaseException("ES alias not found for purpose/persona " + params.getPurpose());
         }
     }
+
+    private void accessControlExclusiveDsl(IndexSearchParams params, String aliasName) {
+
+        List<Map<String, Object>> mustClauses = new ArrayList<>();
+        Map<String, Object> clientQuery = (Map<String, Object>) params.getDsl().get("query");
+
+        mustClauses.add(clientQuery);
+
+        List<Map<String, Object>>filterClauses = new ArrayList<>();
+        filterClauses.add(getMap("terms", getMap("_index", Collections.singletonList(aliasName))));
+
+        Map<String, Object> boolQuery = new HashMap<>();
+        boolQuery.put("must", mustClauses);
+        boolQuery.put("filter",filterClauses);
+
+        List<Map<String, Object>> shouldClauses = new ArrayList<>();
+        shouldClauses.add(getMap("bool", boolQuery));
+        shouldClauses.add(getStaticBoolQuery());
+
+        Map<String, Object> topBoolQuery = getMap("bool", getMap("should", shouldClauses));
+
+        Map copyOfDsl = new HashMap(params.getDsl());
+        copyOfDsl.put("query", topBoolQuery);
+
+        params.setDsl(copyOfDsl);
+    }
+
+    private Map<String, Object> getStaticBoolQuery() {
+        List<Map<String, Object>> mustClauses = new ArrayList<>();
+        Map<String, Object> mustClause = getMap("bool", getMap("should", Arrays.asList(
+                getMap("term", getMap("daapVisibility", "Public")),
+                getMap("term", getMap("daapVisibility", "Protected"))
+        )));
+        mustClauses.add(mustClause);
+
+        List<Map<String, Object>>filterClauses = new ArrayList<>();
+        filterClauses.add(getMap("terms", getMap("_index", Collections.singletonList(VERTEX_INDEX_NAME))));
+
+        Map<String, Object> boolQuery = new HashMap<>();
+        boolQuery.put("must", mustClauses);
+        boolQuery.put("filter", filterClauses);
+
+        return getMap("bool", boolQuery);
+    }
 }

repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java

@@ -341,16 +341,21 @@ public static boolean getRemovePropagations(AtlasVertex classificationVertex) {
         return ret;
     }
 
-    public static boolean getRestrictPropagationThroughLineage(AtlasVertex classificationVertex) {
-        boolean ret = false;
+    public static boolean getRestrictPropagation(AtlasVertex classificationVertex, String propertyName) {
+        if (classificationVertex == null) {
+            return false;
+        }
+        Boolean restrictPropagation = AtlasGraphUtilsV2.getEncodedProperty(classificationVertex, propertyName, Boolean.class);
 
-        if (classificationVertex != null) {
-            Boolean restrictPropagationThroughLineage = AtlasGraphUtilsV2.getEncodedProperty(classificationVertex, CLASSIFICATION_VERTEX_RESTRICT_PROPAGATE_THROUGH_LINEAGE, Boolean.class);
+        return restrictPropagation != null && restrictPropagation;
+    }
 
-            ret = (restrictPropagationThroughLineage == null) ? false : restrictPropagationThroughLineage;
-        }
+    public static boolean getRestrictPropagationThroughLineage(AtlasVertex classificationVertex) {
+        return getRestrictPropagation(classificationVertex,CLASSIFICATION_VERTEX_RESTRICT_PROPAGATE_THROUGH_LINEAGE);
+    }
 
-        return ret;
+    public static boolean getRestrictPropagationThroughHierarchy(AtlasVertex classificationVertex) {
+        return getRestrictPropagation(classificationVertex,CLASSIFICATION_VERTEX_RESTRICT_PROPAGATE_THROUGH_HIERARCHY);
     }
 
     public static AtlasVertex getClassificationVertex(AtlasVertex entityVertex, String classificationName) {

repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java

@@ -1312,10 +1312,10 @@ public void createAndQueueTask(String taskType, AtlasVertex entityVertex, String
         createAndQueueTaskWithoutCheck(taskType, entityVertex, classificationVertexId, relationshipGuid);
     }
 
-    public void createAndQueueTaskWithoutCheck(String taskType, AtlasVertex entityVertex, String classificationVertexId, String relationshipGuid, Boolean currentRestrictPropagationThroughLineage) throws AtlasBaseException {
+    public void createAndQueueTaskWithoutCheck(String taskType, AtlasVertex entityVertex, String classificationVertexId, String relationshipGuid, Boolean currentRestrictPropagationThroughLineage,Boolean currentRestrictPropogationThroughHierarchy) throws AtlasBaseException {
         String              currentUser = RequestContext.getCurrentUser();
         String              entityGuid  = GraphHelper.getGuid(entityVertex);
-        Map<String, Object> taskParams  = ClassificationTask.toParameters(entityGuid, classificationVertexId, relationshipGuid, currentRestrictPropagationThroughLineage);
+        Map<String, Object> taskParams  = ClassificationTask.toParameters(entityGuid, classificationVertexId, relationshipGuid, currentRestrictPropagationThroughLineage,currentRestrictPropogationThroughHierarchy);
         AtlasTask           task        = taskManagement.createTask(taskType, currentUser, taskParams, classificationVertexId, entityGuid);
 
         AtlasGraphUtilsV2.addEncodedProperty(entityVertex, PENDING_TASKS_PROPERTY_KEY, task.getGuid());

repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java

@@ -248,7 +248,7 @@ private void commitChanges(String entityGuid, String typeName, Map<String, List<
             operationListMap.clear();
         }
 
-        private Map<String, List<AtlasClassification>> computeChanges(AtlasEntityHeader incomingEntityHeader, AtlasEntityHeader entityToBeUpdated) {
+        private Map<String, List<AtlasClassification>> computeChanges(AtlasEntityHeader incomingEntityHeader, AtlasEntityHeader entityToBeUpdated) throws AtlasBaseException {
             if (incomingEntityHeader == null || entityToBeUpdated == null) {
                 return null;
             }
@@ -270,6 +270,8 @@ private Map<String, List<AtlasClassification>> computeChanges(AtlasEntityHeader
             return operationListMap;
         }
 
+
+
         private void bucket(String op, Map<String, List<AtlasClassification>> operationListMap, List<AtlasClassification> results) {
             if (CollectionUtils.isEmpty(results)) {
                 return;
@@ -282,7 +284,6 @@ private void addClassifications(String entityGuid, String typeName, List<AtlasCl
             if (CollectionUtils.isEmpty(list)) {
                 return;
             }
-
             String classificationNames = getClassificationNames(list);
             try {
                 entitiesStore.addClassifications(entityGuid, list);
@@ -296,7 +297,6 @@ private void updateClassifications(String entityGuid, String typeName, List<Atla
             if (CollectionUtils.isEmpty(list)) {
                 return;
             }
-
             String classificationNames = getClassificationNames(list);
             try {
                 entitiesStore.updateClassifications(entityGuid, list);