From 209d05e887b45bc70ce96ebe726db3f8a6503217 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Thu, 28 Mar 2024 13:24:26 +0530 Subject: [PATCH] PR Review Changes Done --- .../apache/atlas/repository/Constants.java | 3 +-- .../atlas/repository/graph/GraphHelper.java | 27 +++++++------------ .../store/graph/v1/DeleteHandlerV1.java | 15 ----------- .../store/graph/v2/AtlasEntityComparator.java | 6 ----- .../store/graph/v2/AtlasEntityStoreV2.java | 3 --- .../graph/v2/ClassificationAssociator.java | 4 --- .../store/graph/v2/EntityGraphMapper.java | 12 ++++----- .../store/graph/v2/EntityGraphRetriever.java | 2 +- 8 files changed, 18 insertions(+), 54 deletions(-) diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index e3169f6085..8d4d47ea57 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -25,7 +25,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.File; import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; @@ -379,7 +378,7 @@ public enum SupportedFileExtensions { XLSX, XLS, CSV } public static final String CLASSIFICATION_PROPAGATION_MODE_RESTRICT_HIERARCHY ="RESTRICT_HIERARCHY"; - public static final HashMap> CLASSIFICATION_PROPAGATION_EXCLUSION_MAP = new HashMap>(){{ + public static final HashMap> CLASSIFICATION_PROPAGATION_MODE_LABELS_MAP = new HashMap>(){{ put(CLASSIFICATION_PROPAGATION_MODE_RESTRICT_LINEAGE, new ArrayList<>( Arrays.asList(CATALOG_PROCESS_INPUT_RELATIONSHIP_LABEL, CATALOG_PROCESS_OUTPUT_RELATIONSHIP_LABEL, diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index c82071c9fd..987ceec296 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -341,28 +341,21 @@ public static boolean getRemovePropagations(AtlasVertex classificationVertex) { return ret; } - public static boolean getRestrictPropagationThroughLineage(AtlasVertex classificationVertex) { - boolean ret = false; - - if (classificationVertex != null) { - Boolean restrictPropagationThroughLineage = AtlasGraphUtilsV2.getEncodedProperty(classificationVertex, CLASSIFICATION_VERTEX_RESTRICT_PROPAGATE_THROUGH_LINEAGE, Boolean.class); - - ret = (restrictPropagationThroughLineage == null) ? false : restrictPropagationThroughLineage; + public static boolean getRestrictPropagation(AtlasVertex classificationVertex, String propertyName) { + if (classificationVertex == null) { + return false; } + Boolean restrictPropagation = AtlasGraphUtilsV2.getEncodedProperty(classificationVertex, propertyName, Boolean.class); - return ret; + return restrictPropagation != null && restrictPropagation; } - public static boolean getRestrictPropagationThroughHierarchy(AtlasVertex classificationVertex) { - boolean ret = false; - - if (classificationVertex != null) { - Boolean restrictPropagationThroughHierarchy = AtlasGraphUtilsV2.getEncodedProperty(classificationVertex, CLASSIFICATION_VERTEX_RESTRICT_PROPAGATE_THROUGH_HIERARCHY, Boolean.class); - - ret = (restrictPropagationThroughHierarchy == null) ? false : restrictPropagationThroughHierarchy; - } + public static boolean getRestrictPropagationThroughLineage(AtlasVertex classificationVertex) { + return getRestrictPropagation(classificationVertex,CLASSIFICATION_VERTEX_RESTRICT_PROPAGATE_THROUGH_LINEAGE); + } - return ret; + public static boolean getRestrictPropagationThroughHierarchy(AtlasVertex classificationVertex) { + return getRestrictPropagation(classificationVertex,CLASSIFICATION_VERTEX_RESTRICT_PROPAGATE_THROUGH_HIERARCHY); } public static AtlasVertex getClassificationVertex(AtlasVertex entityVertex, String classificationName) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java index 9da227249a..a4358e2967 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java @@ -1312,21 +1312,6 @@ public void createAndQueueTask(String taskType, AtlasVertex entityVertex, String createAndQueueTaskWithoutCheck(taskType, entityVertex, classificationVertexId, relationshipGuid); } - public void createAndQueueTask(String taskType, AtlasVertex entityVertex, String classificationVertexId, String relationshipGuid, Boolean currentRestrictPropagationThroughLineage, Boolean currentRestrictPropagationThroughHierarchy) throws AtlasBaseException { - if (!CLASSIFICATION_PROPAGATION_DELETE.equals(taskType) && skipClassificationTaskCreation(classificationVertexId)) { - LOG.info("Task is already scheduled for classification id {}, no need to schedule task for vertex {}", classificationVertexId, entityVertex.getIdForDisplay()); - return; - } - - String currentUser = RequestContext.getCurrentUser(); - String entityGuid = GraphHelper.getGuid(entityVertex); - Map taskParams = ClassificationTask.toParameters(entityGuid, classificationVertexId, relationshipGuid, currentRestrictPropagationThroughLineage, currentRestrictPropagationThroughHierarchy); - AtlasTask task = taskManagement.createTask(taskType, currentUser, taskParams, classificationVertexId, entityGuid); - - AtlasGraphUtilsV2.addEncodedProperty(entityVertex, PENDING_TASKS_PROPERTY_KEY, task.getGuid()); - - RequestContext.get().queueTask(task); - } public void createAndQueueTaskWithoutCheck(String taskType, AtlasVertex entityVertex, String classificationVertexId, String relationshipGuid, Boolean currentRestrictPropagationThroughLineage,Boolean currentRestrictPropogationThroughHierarchy) throws AtlasBaseException { String currentUser = RequestContext.getCurrentUser(); String entityGuid = GraphHelper.getGuid(entityVertex); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityComparator.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityComparator.java index a1b2544bae..ce38f4898e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityComparator.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityComparator.java @@ -60,12 +60,6 @@ public AtlasEntityDiffResult getDiffResult(AtlasEntity updatedEntity, AtlasVerte return getDiffResult(updatedEntity, null, storedVertex, findOnlyFirstDiff); } - private void verifyClassificationsPropagationMode(List incomingClassifications) throws AtlasBaseException { - for(AtlasClassification incomingClassification : incomingClassifications){ - entityRetriever.determinePropagationMode(incomingClassification.getRestrictPropagationThroughLineage(),incomingClassification.getRestrictPropagationThroughHierachy()); - } - } - private AtlasEntityDiffResult getDiffResult(AtlasEntity updatedEntity, AtlasEntity storedEntity, AtlasVertex storedVertex, boolean findOnlyFirstDiff) throws AtlasBaseException { AtlasEntity diffEntity = new AtlasEntity(updatedEntity.getTypeName()); AtlasEntityType entityType = typeRegistry.getEntityTypeByName(updatedEntity.getTypeName()); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 9e26f6988a..6e3487f8d3 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -1570,9 +1570,6 @@ private EntityMutationContext preCreateOrUpdate(EntityStream entityStream, Entit String guid = element.getKey(); AtlasEntity entity = entityStream.getByGuid(guid); - if(Objects.nonNull(entity) && Objects.nonNull(entity.getClassifications())) { - entityRetriever.verifyClassificationsPropagationMode(entity.getClassifications()); - } if (entity != null) { // entity would be null if guid is not in the stream but referenced by an entity in the stream AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName()); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java index 63fd40d3c4..6cd9ac4e4b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java @@ -257,8 +257,6 @@ private Map> computeChanges(AtlasEntityHeader List incomingClassifications = listOps.filter(incomingEntityHeader.getGuid(), incomingEntityHeader.getClassifications()); List entityClassifications = listOps.filter(entityToBeUpdated.getGuid(), entityToBeUpdated.getClassifications()); - entityRetriever.verifyClassificationsPropagationMode(incomingClassifications); - if (CollectionUtils.isEmpty(incomingClassifications) && CollectionUtils.isEmpty(entityClassifications)) { return null; } @@ -286,7 +284,6 @@ private void addClassifications(String entityGuid, String typeName, List entitiesToPropagateTo = null; Map> addedClassifications = new HashMap<>(); List addClassifications = new ArrayList<>(classifications.size()); - + entityRetriever.verifyClassificationsPropagationMode(classifications); for (AtlasClassification c : classifications) { AtlasClassification classification = new AtlasClassification(c); String classificationName = classification.getTypeName(); @@ -3077,7 +3077,7 @@ public void addClassifications(final EntityMutationContext context, String guid, String propagationMode; propagationMode = entityRetriever.determinePropagationMode(classification.getRestrictPropagationThroughLineage(),classification.getRestrictPropagationThroughHierachy()); Boolean toExclude = propagationMode == CLASSIFICATION_PROPAGATION_MODE_RESTRICT_LINEAGE ? true : false; - entitiesToPropagateTo = entityRetriever.getImpactedVerticesV2(entityVertex, CLASSIFICATION_PROPAGATION_EXCLUSION_MAP.get(propagationMode),toExclude); + entitiesToPropagateTo = entityRetriever.getImpactedVerticesV2(entityVertex, CLASSIFICATION_PROPAGATION_MODE_LABELS_MAP.get(propagationMode),toExclude); } if (CollectionUtils.isNotEmpty(entitiesToPropagateTo)) { @@ -3170,7 +3170,7 @@ public List propagateClassification(String entityGuid, String classifica String propagationMode = entityRetriever.determinePropagationMode(currentRestrictPropagationThroughLineage, currentRestrictPropagationThroughHierarchy); - List edgeLabelsToCheck = CLASSIFICATION_PROPAGATION_EXCLUSION_MAP.get(propagationMode); + List edgeLabelsToCheck = CLASSIFICATION_PROPAGATION_MODE_LABELS_MAP.get(propagationMode); Boolean toExclude = propagationMode == CLASSIFICATION_PROPAGATION_MODE_RESTRICT_LINEAGE ? true:false; List impactedVertices = entityRetriever.getIncludedImpactedVerticesV2(entityVertex, relationshipGuid, classificationVertexId, edgeLabelsToCheck,toExclude); @@ -3613,7 +3613,7 @@ public void updateClassifications(EntityMutationContext context, String guid, Li propagationMode = CLASSIFICATION_PROPAGATION_MODE_DEFAULT; } Boolean toExclude = propagationMode == CLASSIFICATION_VERTEX_RESTRICT_PROPAGATE_THROUGH_LINEAGE ? true : false; - entitiesToPropagateTo = entityRetriever.getImpactedVerticesV2(entityVertex, null, classificationVertex.getIdForDisplay(), CLASSIFICATION_PROPAGATION_EXCLUSION_MAP.get(propagationMode),toExclude); + entitiesToPropagateTo = entityRetriever.getImpactedVerticesV2(entityVertex, null, classificationVertex.getIdForDisplay(), CLASSIFICATION_PROPAGATION_MODE_LABELS_MAP.get(propagationMode),toExclude); } if (CollectionUtils.isNotEmpty(entitiesToPropagateTo)) { @@ -3906,7 +3906,7 @@ public void classificationRefreshPropagation(String classificationId) throws Atl List verticesIdsToAddClassification = new ArrayList<>(); List propagatedVerticesIdWithoutEdge = entityRetriever.getImpactedVerticesIdsClassificationAttached(sourceEntityVertex , classificationId, - CLASSIFICATION_PROPAGATION_EXCLUSION_MAP.get(propagationMode),toExclude, verticesIdsToAddClassification); + CLASSIFICATION_PROPAGATION_MODE_LABELS_MAP.get(propagationMode),toExclude, verticesIdsToAddClassification); LOG.info("To add classification with id {} to {} vertices", classificationId, verticesIdsToAddClassification.size()); @@ -3954,7 +3954,7 @@ private void processClassificationDeleteOnlyPropagation(AtlasVertex currentClass LOG.info("Traversed {} vertices including edge with relationship GUID {} for classification vertex {}", propagatedVerticesIds.size(), relationshipGuid, classificationId); List propagatedVerticesIdWithoutEdge = entityRetriever.getImpactedVerticesIds(sourceEntityVertex, relationshipGuid , classificationId, - CLASSIFICATION_PROPAGATION_EXCLUSION_MAP.get(propagationMode),toExclude); + CLASSIFICATION_PROPAGATION_MODE_LABELS_MAP.get(propagationMode),toExclude); LOG.info("Traversed {} vertices except edge with relationship GUID {} for classification vertex {}", propagatedVerticesIdWithoutEdge.size(), relationshipGuid, classificationId); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 74956d0d0b..f27e698741 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -535,7 +535,7 @@ public Map> getClassificationPropagatedEntitiesMa propagationMode = determinePropagationMode(restrictPropagationThroughLineage,restrictPropagationThroughHierarchy); Boolean toExclude = propagationMode == CLASSIFICATION_PROPAGATION_MODE_RESTRICT_LINEAGE ? true : false; List entitiesPropagatingTo = getImpactedVerticesV2(sourceEntityVertex, relationshipGuidToExclude, - classificationId, CLASSIFICATION_PROPAGATION_EXCLUSION_MAP.get(propagationMode),toExclude); + classificationId, CLASSIFICATION_PROPAGATION_MODE_LABELS_MAP.get(propagationMode),toExclude); LOG.info("Traversed {} vertices for Classification vertex id {} excluding RelationShip GUID {}", entitiesPropagatingTo.size(), classificationId, relationshipGuidToExclude);