From 46e0f67230504c60014723105770788d467b9cab Mon Sep 17 00:00:00 2001 From: aarshi Date: Wed, 10 Jan 2024 12:36:00 +0530 Subject: [PATCH 1/7] Add utils for removeRelationshipAttributes --- .../atlas/model/instance/AtlasEntity.java | 23 +++++++++++++++++++ .../graph/EntityGraphDiscoveryContext.java | 9 ++++++++ 2 files changed, 32 insertions(+) diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java index 87943e55fe..5349fb1c3c 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java @@ -364,6 +364,12 @@ public Object getAppendRelationshipAttribute(String name) { return a != null ? a.get(name) : null; } + public Object getRemoveRelationshipAttribute(String name) { + Map a = this.removeRelationshipAttributes; + + return a != null ? a.get(name) : null; + } + public boolean hasRelationshipAttribute(String name) { Map r = this.relationshipAttributes; @@ -404,7 +410,24 @@ public void setRemoveRelationshipAttributes(Map removeRelationsh this.removeRelationshipAttributes = removeRelationshipAttributes; } + public boolean hasRemoveRelationshipAttribute(String name) { + Map r = this.relationshipAttributes; + + return r != null ? r.containsKey(name) : false; + } + + public void setRemoveRelationshipAttribute(String name, Object value) { + Map r = this.removeRelationshipAttributes; + if (r != null) { + r.put(name, value); + } else { + r = new HashMap<>(); + r.put(name, value); + + this.removeRelationshipAttributes = r; + } + } public Map getCustomAttributes() { return customAttributes; } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityGraphDiscoveryContext.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityGraphDiscoveryContext.java index af07f7ae1d..de85ff3ee3 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityGraphDiscoveryContext.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/EntityGraphDiscoveryContext.java @@ -45,6 +45,7 @@ public final class EntityGraphDiscoveryContext { private final Set localGuids = new HashSet<>(); private boolean isAppendRelationshipAttributeVisited; + private boolean isRemoveRelationshipAttributeVisited; public EntityGraphDiscoveryContext(AtlasTypeRegistry typeRegistry, EntityStream entityStream) { this.typeRegistry = typeRegistry; @@ -164,4 +165,12 @@ public boolean isAppendRelationshipAttributeVisited() { public void setAppendRelationshipAttributeVisited(boolean appendRelationshipAttributeVisited) { isAppendRelationshipAttributeVisited = appendRelationshipAttributeVisited; } + + public boolean isRemoveRelationshipAttributeVisited() { + return isRemoveRelationshipAttributeVisited; + } + + public void setRemoveRelationshipAttributeVisited(boolean removeRelationshipAttributeVisited) { + isRemoveRelationshipAttributeVisited = removeRelationshipAttributeVisited; + } } From 33855d8e283d353631a0cc16da4e3be29f38e630 Mon Sep 17 00:00:00 2001 From: aarshi Date: Wed, 10 Jan 2024 14:05:13 +0530 Subject: [PATCH 2/7] processRemoveRelationAttributes --- .../graph/v2/AtlasEntityGraphDiscoveryV2.java | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityGraphDiscoveryV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityGraphDiscoveryV2.java index 70da6fe8ef..e3bf08e80a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityGraphDiscoveryV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityGraphDiscoveryV2.java @@ -376,15 +376,28 @@ private void visitRelationships(AtlasEntityType entityType, AtlasEntity entity, visitAttribute(attribute.getAttributeType(), attrVal, entity.getGuid()); visitedAttributes.add(attrName); - } else if (entity.hasAppendRelationshipAttribute(attrName)) { - discoveryContext.setAppendRelationshipAttributeVisited(true); - Object attrVal = entity.getAppendRelationshipAttribute(attrName); - String relationshipType = AtlasEntityUtil.getRelationshipType(attrVal); - AtlasAttribute attribute = entityType.getRelationshipAttribute(attrName, relationshipType); + } else { + if (entity.hasAppendRelationshipAttribute(attrName)) { + discoveryContext.setAppendRelationshipAttributeVisited(true); + Object attrVal = entity.getAppendRelationshipAttribute(attrName); + String relationshipType = AtlasEntityUtil.getRelationshipType(attrVal); + AtlasAttribute attribute = entityType.getRelationshipAttribute(attrName, relationshipType); - visitAttribute(attribute.getAttributeType(), attrVal, entity.getGuid()); + visitAttribute(attribute.getAttributeType(), attrVal, entity.getGuid()); - visitedAttributes.add(attrName); + visitedAttributes.add(attrName); + } + + if (entity.hasRemoveRelationshipAttribute(attrName)) { + discoveryContext.setRemoveRelationshipAttributeVisited(true); + Object attrVal = entity.getRemoveRelationshipAttribute(attrName); + String relationshipType = AtlasEntityUtil.getRelationshipType(attrVal); + AtlasAttribute attribute = entityType.getRelationshipAttribute(attrName, relationshipType); + + visitAttribute(attribute.getAttributeType(), attrVal, entity.getGuid()); + + visitedAttributes.add(attrName); + } } } } From 8e82dc181a4c817ff68c825ef9d57b444b83f9dc Mon Sep 17 00:00:00 2001 From: aarshi Date: Wed, 10 Jan 2024 14:05:46 +0530 Subject: [PATCH 3/7] set removeRelationshipAttributes in context --- .../atlas/repository/store/graph/v2/AtlasEntityStoreV2.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index d870611c01..6e3487f8d3 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -1647,6 +1647,10 @@ private EntityMutationContext preCreateOrUpdate(EntityStream entityStream, Entit context.setUpdatedWithRelationshipAttributes(entity); } + if (discoveryContext.isRemoveRelationshipAttributeVisited() && MapUtils.isNotEmpty(entity.getRemoveRelationshipAttributes())) { + context.setUpdatedWithRemoveRelationshipAttributes(entity); + } + if (isRestoreRequested) { Status currStatus = AtlasGraphUtilsV2.getState(vertex); if (currStatus == Status.DELETED) { From 35aae5aee094f773233236e0d6e244d4ba3d9608 Mon Sep 17 00:00:00 2001 From: aarshi Date: Wed, 10 Jan 2024 14:07:19 +0530 Subject: [PATCH 4/7] Add utils to process removeRelationshipAttributes and minor code refactoring --- .../store/graph/v2/EntityGraphMapper.java | 114 ++++++++++++------ .../store/graph/v2/EntityMutationContext.java | 9 ++ 2 files changed, 85 insertions(+), 38 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 06c8eeff2f..81730f8069 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -346,8 +346,8 @@ public EntityMutationResponse mapAttributesAndClassifications(EntityMutationCont MetricRecorder metric = RequestContext.get().startMetricRecord("mapAttributesAndClassifications"); - EntityMutationResponse resp = new EntityMutationResponse(); - RequestContext reqContext = RequestContext.get(); + EntityMutationResponse resp = new EntityMutationResponse(); + RequestContext reqContext = RequestContext.get(); if (CollectionUtils.isNotEmpty(context.getEntitiesToRestore())) { restoreHandlerV1.restoreEntities(context.getEntitiesToRestore()); @@ -356,6 +356,7 @@ public EntityMutationResponse mapAttributesAndClassifications(EntityMutationCont Collection createdEntities = context.getCreatedEntities(); Collection updatedEntities = context.getUpdatedEntities(); Collection appendEntities = context.getUpdatedEntitiesForAppendRelationshipAttribute(); + Collection removeEntities = context.getEntitiesUpdatedWithRemoveRelationshipAttribute(); if (CollectionUtils.isNotEmpty(createdEntities)) { for (AtlasEntity createdEntity : createdEntities) { @@ -371,7 +372,7 @@ public EntityMutationResponse mapAttributesAndClassifications(EntityMutationCont setCustomAttributes(vertex, createdEntity); setSystemAttributesToEntity(vertex, createdEntity); - resp.addEntity(CREATE, constructHeader(createdEntity, vertex, entityType.getAllAttributes())); + resp.addEntity(CREATE, constructHeader(createdEntity, vertex, entityType.getAllAttributes())); addClassifications(context, guid, createdEntity.getClassifications()); if (MapUtils.isNotEmpty(createdEntity.getBusinessAttributes())) { @@ -478,15 +479,26 @@ public EntityMutationResponse mapAttributesAndClassifications(EntityMutationCont throw baseException; } } - } else if (CollectionUtils.isNotEmpty(appendEntities)) { - for (AtlasEntity entity : appendEntities) { - String guid = entity.getGuid(); - AtlasVertex vertex = context.getVertex(guid); - AtlasEntityType entityType = context.getType(guid); - mapAppendRelationshipAttributes(entity, entityType, vertex, UPDATE, context); + } else { + + if (CollectionUtils.isNotEmpty(appendEntities)) { + for (AtlasEntity entity : appendEntities) { + String guid = entity.getGuid(); + AtlasVertex vertex = context.getVertex(guid); + AtlasEntityType entityType = context.getType(guid); + mapAppendRelationshipAttributes(entity, entityType, vertex, UPDATE, context); + } } - } + if (CollectionUtils.isNotEmpty(removeEntities)) { + for (AtlasEntity entity : removeEntities) { + String guid = entity.getGuid(); + AtlasVertex vertex = context.getVertex(guid); + AtlasEntityType entityType = context.getType(guid); + mapRemoveRelationshipAttributes(entity, entityType, vertex, UPDATE, context); + } + } + } if (CollectionUtils.isNotEmpty(context.getEntitiesToDelete())) { deleteDelegate.getHandler().deleteEntities(context.getEntitiesToDelete()); @@ -1074,11 +1086,11 @@ private void mapRelationshipAttributes(AtlasEntity entity, AtlasEntityType entit private void mapAppendRelationshipAttributes(AtlasEntity entity, AtlasEntityType entityType, AtlasVertex vertex, EntityOperation op, EntityMutationContext context) throws AtlasBaseException { if (LOG.isDebugEnabled()) { - LOG.debug("==> mapRelationshipAttributes({}, {})", op, entity.getTypeName()); + LOG.debug("==> mapAppendRelationshipAttributes({}, {})", op, entity.getTypeName()); } if (MapUtils.isNotEmpty(entity.getAppendRelationshipAttributes())) { - MetricRecorder metric = RequestContext.get().startMetricRecord("mapRelationshipAttributes"); + MetricRecorder metric = RequestContext.get().startMetricRecord("mapAppendRelationshipAttributes"); if (op.equals(CREATE)) { for (String attrName : entityType.getRelationshipAttributes().keySet()) { @@ -1097,7 +1109,7 @@ private void mapAppendRelationshipAttributes(AtlasEntity entity, AtlasEntityType String relationType = AtlasEntityUtil.getRelationshipType(attrValue); AtlasAttribute attribute = entityType.getRelationshipAttribute(attrName, relationType); - mapAppendRelationshipAttribute(attribute, attrValue, vertex, op, context); + mapAttribute(attribute, attrValue, vertex, op, context, true, false); } } } @@ -1108,39 +1120,55 @@ private void mapAppendRelationshipAttributes(AtlasEntity entity, AtlasEntityType } if (LOG.isDebugEnabled()) { - LOG.debug("<== mapRelationshipAttributes({}, {})", op, entity.getTypeName()); + LOG.debug("<== mapAppendRelationshipAttributes({}, {})", op, entity.getTypeName()); } } - private void mapAttribute(AtlasAttribute attribute, Object attrValue, AtlasVertex vertex, EntityOperation op, EntityMutationContext context) throws AtlasBaseException { - boolean isDeletedEntity = context.isDeletedEntity(vertex); - AtlasType attrType = attribute.getAttributeType(); - if (attrValue == null) { - AtlasAttributeDef attributeDef = attribute.getAttributeDef(); - if (attrType.getTypeCategory() == TypeCategory.PRIMITIVE) { - if (attributeDef.getDefaultValue() != null) { - attrValue = attrType.createDefaultValue(attributeDef.getDefaultValue()); - } else if (attributeDef.getIsDefaultValueNull() && ALLOWED_DATATYPES_FOR_DEFAULT_NULL.contains(attribute.getTypeName())) { - attrValue = null; - } else { - if (attribute.getAttributeDef().getIsOptional()) { - attrValue = attrType.createOptionalDefaultValue(); - } else { - attrValue = attrType.createDefaultValue(); + private void mapRemoveRelationshipAttributes(AtlasEntity entity, AtlasEntityType entityType, AtlasVertex vertex, EntityOperation op, + EntityMutationContext context) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("==> mapRemoveRelationshipAttributes({}, {})", op, entity.getTypeName()); + } + + if (MapUtils.isNotEmpty(entity.getRemoveRelationshipAttributes())) { + MetricRecorder metric = RequestContext.get().startMetricRecord("mapRemoveRelationshipAttributes"); + + if (op.equals(CREATE)) { + for (String attrName : entityType.getRelationshipAttributes().keySet()) { + Object attrValue = entity.getRemoveRelationshipAttribute(attrName); + String relationType = AtlasEntityUtil.getRelationshipType(attrValue); + AtlasAttribute attribute = entityType.getRelationshipAttribute(attrName, relationType); + + mapAttribute(attribute, attrValue, vertex, op, context); + } + + } else if (op.equals(UPDATE) || op.equals(PARTIAL_UPDATE)) { + // relationship attributes mapping + for (String attrName : entityType.getRelationshipAttributes().keySet()) { + if (entity.hasRemoveRelationshipAttribute(attrName)) { + Object attrValue = entity.getRemoveRelationshipAttribute(attrName); + String relationType = AtlasEntityUtil.getRelationshipType(attrValue); + AtlasAttribute attribute = entityType.getRelationshipAttribute(attrName, relationType); + + mapAttribute(attribute, attrValue, vertex, op, context, false, true); } } } + + updateModificationMetadata(vertex); + + RequestContext.get().endMetricRecord(metric); } - if (attrType.getTypeCategory() == TypeCategory.PRIMITIVE || attrType.getTypeCategory() == TypeCategory.ENUM) { - mapPrimitiveValue(vertex, attribute, attrValue, isDeletedEntity); - } else { - AttributeMutationContext ctx = new AttributeMutationContext(op, vertex, attribute, attrValue); - mapToVertexByTypeCategory(ctx, context, false); + if (LOG.isDebugEnabled()) { + LOG.debug("<== mapRemoveRelationshipAttributes({}, {})", op, entity.getTypeName()); } } + private void mapAttribute(AtlasAttribute attribute, Object attrValue, AtlasVertex vertex, EntityOperation op, EntityMutationContext context) throws AtlasBaseException { + mapAttribute(attribute, attrValue, vertex, op, context, false, false); + } - private void mapAppendRelationshipAttribute(AtlasAttribute attribute, Object attrValue, AtlasVertex vertex, EntityOperation op, EntityMutationContext context) throws AtlasBaseException { + private void mapAttribute(AtlasAttribute attribute, Object attrValue, AtlasVertex vertex, EntityOperation op, EntityMutationContext context, boolean isAppendOp, boolean isRemoveOp) throws AtlasBaseException { boolean isDeletedEntity = context.isDeletedEntity(vertex); AtlasType attrType = attribute.getAttributeType(); if (attrValue == null) { @@ -1165,11 +1193,11 @@ private void mapAppendRelationshipAttribute(AtlasAttribute attribute, Object att mapPrimitiveValue(vertex, attribute, attrValue, isDeletedEntity); } else { AttributeMutationContext ctx = new AttributeMutationContext(op, vertex, attribute, attrValue); - mapToVertexByTypeCategory(ctx, context, true); + mapToVertexByTypeCategory(ctx, context, isAppendOp, isRemoveOp); } } - private Object mapToVertexByTypeCategory(AttributeMutationContext ctx, EntityMutationContext context, boolean isAppendOperation) throws AtlasBaseException { + private Object mapToVertexByTypeCategory(AttributeMutationContext ctx, EntityMutationContext context, boolean isAppendOp, boolean isRemoveOp) throws AtlasBaseException { if (ctx.getOp() == CREATE && ctx.getValue() == null) { return null; } @@ -1280,9 +1308,14 @@ private Object mapToVertexByTypeCategory(AttributeMutationContext ctx, EntityMut return mapMapValue(ctx, context); case ARRAY: - if (isAppendOperation){ + if (isAppendOp){ return appendArrayValue(ctx, context); } + + if (isRemoveOp){ + return removeArrayValue(ctx, context); + } + return mapArrayValue(ctx, context); default: @@ -2040,6 +2073,11 @@ public List appendArrayValue(AttributeMutationContext ctx, EntityMutationContext return newElementsCreated; } + public List removeArrayValue(AttributeMutationContext ctx, EntityMutationContext context) throws AtlasBaseException { + return new ArrayList(); + + } + private void addEdgesToContext(String guid, List newElementsCreated, List removedElements) { if (newElementsCreated.size() > 0) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityMutationContext.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityMutationContext.java index 895073743c..72c3200809 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityMutationContext.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityMutationContext.java @@ -31,6 +31,7 @@ public class EntityMutationContext { private final List entitiesCreated = new ArrayList<>(); private final List entitiesUpdated = new ArrayList<>(); private final List entitiesUpdatedWithAppendRelationshipAttribute = new ArrayList<>(); + private final List entitiesUpdatedWithRemoveRelationshipAttribute = new ArrayList<>(); private final Map entityVsType = new HashMap<>(); private final Map entityVsVertex = new HashMap<>(); private final Map guidAssignments = new HashMap<>(); @@ -62,6 +63,14 @@ public void setUpdatedWithRelationshipAttributes(AtlasEntity entity){ entitiesUpdatedWithAppendRelationshipAttribute.add(entity); } + public void setUpdatedWithRemoveRelationshipAttributes(AtlasEntity entity){ + entitiesUpdatedWithRemoveRelationshipAttribute.add(entity); + } + + public Collection getEntitiesUpdatedWithRemoveRelationshipAttribute() { + return entitiesUpdatedWithRemoveRelationshipAttribute; + } + public void addUpdated(String internalGuid, AtlasEntity entity, AtlasEntityType type, AtlasVertex atlasVertex) { if (!entityVsVertex.containsKey(internalGuid)) { // if the entity was already created/updated entitiesUpdated.add(entity); From 0247fdebd26e30d1407bcba3d5a088d0048b227f Mon Sep 17 00:00:00 2001 From: aarshi Date: Wed, 10 Jan 2024 14:40:58 +0530 Subject: [PATCH 5/7] code cleanup --- .../store/graph/v2/EntityGraphMapper.java | 71 +++++-------------- 1 file changed, 17 insertions(+), 54 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 81730f8069..3323e9f6c8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -126,7 +126,6 @@ import static org.apache.atlas.repository.graph.GraphHelper.getTraitNames; import static org.apache.atlas.repository.graph.GraphHelper.getTypeName; import static org.apache.atlas.repository.graph.GraphHelper.getTypeNames; -import static org.apache.atlas.repository.graph.GraphHelper.isActive; import static org.apache.atlas.repository.graph.GraphHelper.isPropagationEnabled; import static org.apache.atlas.repository.graph.GraphHelper.isRelationshipEdge; import static org.apache.atlas.repository.graph.GraphHelper.string; @@ -486,7 +485,7 @@ public EntityMutationResponse mapAttributesAndClassifications(EntityMutationCont String guid = entity.getGuid(); AtlasVertex vertex = context.getVertex(guid); AtlasEntityType entityType = context.getType(guid); - mapAppendRelationshipAttributes(entity, entityType, vertex, UPDATE, context); + mapAppendRemoveRelationshipAttributes(entity, entityType, vertex, UPDATE, context, true, false); } } @@ -495,7 +494,7 @@ public EntityMutationResponse mapAttributesAndClassifications(EntityMutationCont String guid = entity.getGuid(); AtlasVertex vertex = context.getVertex(guid); AtlasEntityType entityType = context.getType(guid); - mapRemoveRelationshipAttributes(entity, entityType, vertex, UPDATE, context); + mapAppendRemoveRelationshipAttributes(entity, entityType, vertex, UPDATE, context, false, true); } } } @@ -1083,87 +1082,51 @@ private void mapRelationshipAttributes(AtlasEntity entity, AtlasEntityType entit } } - private void mapAppendRelationshipAttributes(AtlasEntity entity, AtlasEntityType entityType, AtlasVertex vertex, EntityOperation op, - EntityMutationContext context) throws AtlasBaseException { + private void mapAppendRemoveRelationshipAttributes(AtlasEntity entity, AtlasEntityType entityType, AtlasVertex vertex, EntityOperation op, + EntityMutationContext context, boolean isAppendOp, boolean isRemoveOp) throws AtlasBaseException { if (LOG.isDebugEnabled()) { - LOG.debug("==> mapAppendRelationshipAttributes({}, {})", op, entity.getTypeName()); + LOG.debug("==> mapAppendRemoveRelationshipAttributes({}, {})", op, entity.getTypeName()); } - if (MapUtils.isNotEmpty(entity.getAppendRelationshipAttributes())) { - MetricRecorder metric = RequestContext.get().startMetricRecord("mapAppendRelationshipAttributes"); - - if (op.equals(CREATE)) { - for (String attrName : entityType.getRelationshipAttributes().keySet()) { - Object attrValue = entity.getAppendRelationshipAttribute(attrName); - String relationType = AtlasEntityUtil.getRelationshipType(attrValue); - AtlasAttribute attribute = entityType.getRelationshipAttribute(attrName, relationType); - - mapAttribute(attribute, attrValue, vertex, op, context); - } + MetricRecorder metric = RequestContext.get().startMetricRecord("mapAppendRemoveRelationshipAttributes"); - } else if (op.equals(UPDATE) || op.equals(PARTIAL_UPDATE)) { + if (isAppendOp && MapUtils.isNotEmpty(entity.getAppendRelationshipAttributes())) { + if (op.equals(UPDATE) || op.equals(PARTIAL_UPDATE)) { // relationship attributes mapping for (String attrName : entityType.getRelationshipAttributes().keySet()) { if (entity.hasAppendRelationshipAttribute(attrName)) { Object attrValue = entity.getAppendRelationshipAttribute(attrName); String relationType = AtlasEntityUtil.getRelationshipType(attrValue); AtlasAttribute attribute = entityType.getRelationshipAttribute(attrName, relationType); - - mapAttribute(attribute, attrValue, vertex, op, context, true, false); + mapAttribute(attribute, attrValue, vertex, op, context, true, isRemoveOp); } } } - - updateModificationMetadata(vertex); - - RequestContext.get().endMetricRecord(metric); } - if (LOG.isDebugEnabled()) { - LOG.debug("<== mapAppendRelationshipAttributes({}, {})", op, entity.getTypeName()); - } - } - - private void mapRemoveRelationshipAttributes(AtlasEntity entity, AtlasEntityType entityType, AtlasVertex vertex, EntityOperation op, - EntityMutationContext context) throws AtlasBaseException { - if (LOG.isDebugEnabled()) { - LOG.debug("==> mapRemoveRelationshipAttributes({}, {})", op, entity.getTypeName()); - } - - if (MapUtils.isNotEmpty(entity.getRemoveRelationshipAttributes())) { - MetricRecorder metric = RequestContext.get().startMetricRecord("mapRemoveRelationshipAttributes"); - - if (op.equals(CREATE)) { - for (String attrName : entityType.getRelationshipAttributes().keySet()) { - Object attrValue = entity.getRemoveRelationshipAttribute(attrName); - String relationType = AtlasEntityUtil.getRelationshipType(attrValue); - AtlasAttribute attribute = entityType.getRelationshipAttribute(attrName, relationType); - - mapAttribute(attribute, attrValue, vertex, op, context); - } - - } else if (op.equals(UPDATE) || op.equals(PARTIAL_UPDATE)) { + if (isRemoveOp && MapUtils.isNotEmpty(entity.getRemoveRelationshipAttributes())) { + if (op.equals(UPDATE) || op.equals(PARTIAL_UPDATE)) { // relationship attributes mapping for (String attrName : entityType.getRelationshipAttributes().keySet()) { if (entity.hasRemoveRelationshipAttribute(attrName)) { Object attrValue = entity.getRemoveRelationshipAttribute(attrName); String relationType = AtlasEntityUtil.getRelationshipType(attrValue); AtlasAttribute attribute = entityType.getRelationshipAttribute(attrName, relationType); - - mapAttribute(attribute, attrValue, vertex, op, context, false, true); + mapAttribute(attribute, attrValue, vertex, op, context, isAppendOp, true); } } } + } - updateModificationMetadata(vertex); + updateModificationMetadata(vertex); - RequestContext.get().endMetricRecord(metric); - } + RequestContext.get().endMetricRecord(metric); if (LOG.isDebugEnabled()) { - LOG.debug("<== mapRemoveRelationshipAttributes({}, {})", op, entity.getTypeName()); + LOG.debug("<== mapAppendRemoveRelationshipAttributes({}, {})", op, entity.getTypeName()); } } + private void mapAttribute(AtlasAttribute attribute, Object attrValue, AtlasVertex vertex, EntityOperation op, EntityMutationContext context) throws AtlasBaseException { mapAttribute(attribute, attrValue, vertex, op, context, false, false); } From 862888769f892907145c30c357198b7bb24d7c51 Mon Sep 17 00:00:00 2001 From: aarshi Date: Wed, 10 Jan 2024 18:55:00 +0530 Subject: [PATCH 6/7] consume removeRelationshipAttributes --- .../main/java/org/apache/atlas/model/instance/AtlasEntity.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java index 5349fb1c3c..55b50b85e3 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntity.java @@ -411,7 +411,7 @@ public void setRemoveRelationshipAttributes(Map removeRelationsh } public boolean hasRemoveRelationshipAttribute(String name) { - Map r = this.relationshipAttributes; + Map r = this.removeRelationshipAttributes; return r != null ? r.containsKey(name) : false; } From 4295aeac0be4ddeae091a9a3fccf63901db47c8e Mon Sep 17 00:00:00 2001 From: aarshi Date: Wed, 10 Jan 2024 18:56:01 +0530 Subject: [PATCH 7/7] add util to consumeRemoveRelationshipAttributes --- .../store/graph/v2/EntityGraphMapper.java | 103 +++++++++++++++++- 1 file changed, 101 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 3323e9f6c8..db9d92a724 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -1823,7 +1823,7 @@ public List mapArrayValue(AttributeMutationContext ctx, EntityMutationContext co AtlasAttribute inverseRefAttribute = attribute.getInverseRefAttribute(); Cardinality cardinality = attribute.getAttributeDef().getCardinality(); List removedElements = new ArrayList<>(); - List newElementsCreated = new ArrayList<>(); + List newElementsCreated = new ArrayList<>(); List allArrayElements = null; List currentElements; boolean deleteExistingRelations = shouldDeleteExistingRelations(ctx, attribute); @@ -2037,8 +2037,75 @@ public List appendArrayValue(AttributeMutationContext ctx, EntityMutationContext } public List removeArrayValue(AttributeMutationContext ctx, EntityMutationContext context) throws AtlasBaseException { - return new ArrayList(); + if (LOG.isDebugEnabled()) { + LOG.debug("==> removeArrayValue({})", ctx); + } + AtlasAttribute attribute = ctx.getAttribute(); + List elementsDeleted = (List) ctx.getValue(); + AtlasArrayType arrType = (AtlasArrayType) attribute.getAttributeType(); + AtlasType elementType = arrType.getElementType(); + boolean isStructType = (TypeCategory.STRUCT == elementType.getTypeCategory()) || + (TypeCategory.STRUCT == attribute.getDefinedInType().getTypeCategory()); + boolean isReference = isReference(elementType); + boolean isSoftReference = ctx.getAttribute().getAttributeDef().isSoftReferenced(); + Cardinality cardinality = attribute.getAttributeDef().getCardinality(); + List removedElements = new ArrayList<>(); + List entityRelationsDeleted = new ArrayList<>(); + List allArrayElements = null; + List currentElements; + + + if (isReference && !isSoftReference) { + currentElements = (List) getCollectionElementsUsingRelationship(ctx.getReferringVertex(), attribute, isStructType); + } else { + currentElements = (List) getArrayElementsProperty(elementType, isSoftReference, ctx.getReferringVertex(), ctx.getVertexProperty()); + } + + boolean isNewElementsNull = elementsDeleted == null; + + if (isNewElementsNull) { + elementsDeleted = new ArrayList(); + } + + if (cardinality == SET) { + elementsDeleted = (List) elementsDeleted.stream().distinct().collect(Collectors.toList()); + } + + for (int index = 0; index < elementsDeleted.size(); index++) { + AtlasEdge existingEdge = (isSoftReference) ? null : getEdgeAt(currentElements, index, elementType); + AttributeMutationContext arrCtx = new AttributeMutationContext(ctx.getOp(), ctx.getReferringVertex(), ctx.getAttribute(), elementsDeleted.get(index), + ctx.getVertexProperty(), elementType, existingEdge); + + Object newEntry = mapCollectionElementsToVertex(arrCtx, context); + if(newEntry != null) { + entityRelationsDeleted.add(newEntry); + } + } + + removedElements = removeArrayEntries(attribute, (List)entityRelationsDeleted, ctx); + + + switch (ctx.getAttribute().getRelationshipEdgeLabel()) { + case TERM_ASSIGNMENT_LABEL: addMeaningsToEntity(ctx, new ArrayList<>() , removedElements); + break; + + case CATEGORY_TERMS_EDGE_LABEL: addCategoriesToTermEntity(ctx, new ArrayList<>(), removedElements); + break; + + case CATEGORY_PARENT_EDGE_LABEL: addCatParentAttr(ctx, new ArrayList<>(), removedElements); + break; + + case PROCESS_INPUTS: + case PROCESS_OUTPUTS: addEdgesToContext(GraphHelper.getGuid(ctx.referringVertex), new ArrayList<>(), removedElements); + break; + } + + if (LOG.isDebugEnabled()) { + LOG.debug("<== removeArrayValue({})", ctx); + } + + return allArrayElements; } private void addEdgesToContext(String guid, List newElementsCreated, List removedElements) { @@ -2747,6 +2814,38 @@ private List removeUnusedArrayEntries(AtlasAttribute attribute, List< return Collections.emptyList(); } + + private List removeArrayEntries(AtlasAttribute attribute, List tobeDeletedEntries, AttributeMutationContext ctx) throws AtlasBaseException { + if (CollectionUtils.isNotEmpty(tobeDeletedEntries)) { + AtlasType entryType = ((AtlasArrayType) attribute.getAttributeType()).getElementType(); + AtlasVertex entityVertex = ctx.getReferringVertex(); + + if (isReference(entryType)) { + // Collection edgesToRemove = CollectionUtils.subtract(currentEntries, newEntries); + + if (CollectionUtils.isNotEmpty(tobeDeletedEntries)) { + List additionalElements = new ArrayList<>(); + + for (AtlasEdge edge : tobeDeletedEntries) { + if (getStatus(edge) == DELETED ) { + continue; + } + + boolean deleted = deleteDelegate.getHandler().deleteEdgeReference(edge, entryType.getTypeCategory(), attribute.isOwnedRef(), + true, attribute.getRelationshipEdgeDirection(), entityVertex); + + if (!deleted) { + additionalElements.add(edge); + } + } + + return additionalElements; + } + } + } + + return Collections.emptyList(); + } private void setArrayElementsProperty(AtlasType elementType, boolean isSoftReference, AtlasVertex vertex, String vertexPropertyName, List allValues, List currentValues, Cardinality cardinality) { boolean isArrayOfPrimitiveType = elementType.getTypeCategory().equals(TypeCategory.PRIMITIVE); boolean isArrayOfEnum = elementType.getTypeCategory().equals(TypeCategory.ENUM);