From a34d552af23dfc0d51933f051d281a69833b20e6 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Fri, 19 Apr 2024 13:50:16 +0530 Subject: [PATCH 1/3] Removed Column Lineage --- .../main/java/org/apache/atlas/repository/Constants.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index e10796992d..fe46e5a460 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -378,7 +378,6 @@ public enum SupportedFileExtensions { XLSX, XLS, CSV } public static final String CATALOG_PROCESS_INPUT_RELATIONSHIP_LABEL = "__Process.inputs"; public static final String CATALOG_PROCESS_OUTPUT_RELATIONSHIP_LABEL = "__Process.outputs"; - public static final String COLUMN_LINEAGE_RELATIONSHIP_LABEL = "__Process.columnProcesses"; public static final String CLASSIFICATION_PROPAGATION_MODE_DEFAULT ="DEFAULT"; public static final String CLASSIFICATION_PROPAGATION_MODE_RESTRICT_LINEAGE ="RESTRICT_LINEAGE"; @@ -388,14 +387,12 @@ public enum SupportedFileExtensions { XLSX, XLS, CSV } public static final HashMap> CLASSIFICATION_PROPAGATION_MODE_LABELS_MAP = new HashMap>(){{ put(CLASSIFICATION_PROPAGATION_MODE_RESTRICT_LINEAGE, new ArrayList<>( Arrays.asList(CATALOG_PROCESS_INPUT_RELATIONSHIP_LABEL, - CATALOG_PROCESS_OUTPUT_RELATIONSHIP_LABEL, - COLUMN_LINEAGE_RELATIONSHIP_LABEL + CATALOG_PROCESS_OUTPUT_RELATIONSHIP_LABEL ))); put(CLASSIFICATION_PROPAGATION_MODE_DEFAULT, null); put(CLASSIFICATION_PROPAGATION_MODE_RESTRICT_HIERARCHY, new ArrayList<>( Arrays.asList(CATALOG_PROCESS_INPUT_RELATIONSHIP_LABEL, - CATALOG_PROCESS_OUTPUT_RELATIONSHIP_LABEL, - COLUMN_LINEAGE_RELATIONSHIP_LABEL + CATALOG_PROCESS_OUTPUT_RELATIONSHIP_LABEL ))); }}; From 0ec7b748cced0b17a25ff6781e819e97fb5bbfb1 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Tue, 30 Apr 2024 13:44:30 +0530 Subject: [PATCH 2/3] Fixed the updatedTagPropagation logic that was running the compute propagatedEntityVertices logic even when it should not have --- .../repository/store/graph/v2/EntityGraphMapper.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index a95ca20f0c..4f7ceb0588 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -137,8 +137,7 @@ import static org.apache.atlas.repository.graph.GraphHelper.getPropagatableClassifications; import static org.apache.atlas.repository.graph.GraphHelper.getClassificationEntityGuid; import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.*; -import static org.apache.atlas.repository.store.graph.v2.tasks.ClassificationPropagateTaskFactory.CLASSIFICATION_PROPAGATION_ADD; -import static org.apache.atlas.repository.store.graph.v2.tasks.ClassificationPropagateTaskFactory.CLASSIFICATION_PROPAGATION_DELETE; +import static org.apache.atlas.repository.store.graph.v2.tasks.ClassificationPropagateTaskFactory.*; import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.IN; import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.OUT; import static org.apache.atlas.type.Constants.PENDING_TASKS_PROPERTY_KEY; @@ -3587,11 +3586,14 @@ public void updateClassifications(EntityMutationContext context, String guid, Li taskManagement != null && DEFERRED_ACTION_ENABLED) { String propagationType = CLASSIFICATION_PROPAGATION_ADD; - if (removePropagation || !updatedTagPropagation) - { + if(currentRestrictPropagationThroughLineage != updatedRestrictPropagationThroughLineage || currentRestrictPropagationThroughHierarchy != updatedRestrictPropagationThroughHierarchy){ + propagationType = CLASSIFICATION_REFRESH_PROPAGATION; + } + if (removePropagation || !updatedTagPropagation) { propagationType = CLASSIFICATION_PROPAGATION_DELETE; } createAndQueueTask(propagationType, entityVertex, classificationVertex.getIdForDisplay(), currentRestrictPropagationThroughLineage,currentRestrictPropagationThroughHierarchy); + updatedTagPropagation = null; } // compute propagatedEntityVertices once and use it for subsequent iterations and notifications From 160e7bd8a8b1e382b5c9f5bca29daa0a92e5d6dc Mon Sep 17 00:00:00 2001 From: hr2904 Date: Tue, 30 Apr 2024 16:35:45 +0530 Subject: [PATCH 3/3] edge case missed Incase all options are true error should be thrown --- .../repository/store/graph/v2/EntityGraphMapper.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 4f7ceb0588..a5bf1de5d0 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -3579,7 +3579,7 @@ public void updateClassifications(EntityMutationContext context, String guid, Li Boolean updatedRestrictPropagationThroughLineage = classification.getRestrictPropagationThroughLineage(); Boolean currentRestrictPropagationThroughHierarchy = currentClassification.getRestrictPropagationThroughHierarchy(); Boolean updatedRestrictPropagationThroughHierarchy = classification.getRestrictPropagationThroughHierarchy(); - + String propagationMode = entityRetriever.determinePropagationMode(updatedRestrictPropagationThroughLineage, updatedRestrictPropagationThroughHierarchy); if ((!Objects.equals(updatedRemovePropagations, currentRemovePropagations) || !Objects.equals(currentTagPropagation, updatedTagPropagation) || !Objects.equals(currentRestrictPropagationThroughLineage, updatedRestrictPropagationThroughLineage)) && @@ -3607,11 +3607,7 @@ public void updateClassifications(EntityMutationContext context, String guid, Li deleteDelegate.getHandler().removeTagPropagation(classificationVertex); } if (CollectionUtils.isEmpty(entitiesToPropagateTo)) { - String propagationMode; - if (updatedRemovePropagations !=null) { - propagationMode = entityRetriever.determinePropagationMode(updatedRestrictPropagationThroughLineage, updatedRestrictPropagationThroughHierarchy); - } - else{ + if (updatedRemovePropagations ==null) { propagationMode = CLASSIFICATION_PROPAGATION_MODE_DEFAULT; } Boolean toExclude = propagationMode == CLASSIFICATION_VERTEX_RESTRICT_PROPAGATE_THROUGH_LINEAGE ? true : false;