From 95ff1ac0d0f4d63d0346b0d4a87e2b9cc05fa502 Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 22 Jan 2024 15:29:47 +0530 Subject: [PATCH 1/2] Optimise append operation --- .../java/org/apache/atlas/AtlasErrorCode.java | 3 +- .../graph/v2/AtlasEntityGraphDiscoveryV2.java | 6 + .../graph/v2/AttributeMutationContext.java | 5 + .../store/graph/v2/EntityGraphMapper.java | 112 +++++++++--------- .../java/org/apache/atlas/RequestContext.java | 5 + 5 files changed, 77 insertions(+), 54 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java index 04d72bdfb2..311ea39116 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java +++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java @@ -184,6 +184,7 @@ public enum AtlasErrorCode { PAGINATION_CAN_ONLY_BE_USED_WITH_DEPTH_ONE(400, "ATLAS-400-00-103", "Pagination can be used only when depth is 1"), CANT_CALCULATE_VERTEX_COUNTS_WITHOUT_PAGINATION(400, "ATLAS-400-00-104", "Vertex counts can't be calculated without pagination"), FORBIDDEN_TYPENAME(400,"ATLAS-400-00-107", "Forbidden type: Can not pass builtin type {0}"), + ATTRIBUTE_ALREADY_EXISTS_IN_RELATIONSHIP_ATTRIBUTE(400, "ATLAS-400-00-108", "{0}: attribute already exists in relationshipAttributes"), // All Not found enums go here TYPE_NAME_NOT_FOUND(404, "ATLAS-404-00-001", "Given typename {0} was invalid"), @@ -213,6 +214,7 @@ public enum AtlasErrorCode { TASK_NOT_FOUND(404, "ATLAS-404-00-018", "Given task guid {0} is invalid/not found"), RESOURCE_NOT_FOUND(404, "ATLAS-404-00-019", "{0} not found"), INDEX_NOT_FOUND(404, "ATLAS-404-00-020", "ES index {0} not found"), + RELATIONSHIP_DOES_NOT_EXIST(404, "ATLAS-409-00-0021", "relationship {0} does not exist between entities {1} and {2}"), METHOD_NOT_ALLOWED(405, "ATLAS-405-00-001", "Error 405 - The request method {0} is inappropriate for the URL: {1}"), DELETE_TAG_PROPAGATION_NOT_ALLOWED(406, "ATLAS-406-00-001", "Classification delete is not allowed; Add/Update classification propagation is in queue for classification: {0} and entity: {1}. Please try again"), @@ -237,7 +239,6 @@ public enum AtlasErrorCode { CATEGORY_PARENT_FROM_OTHER_GLOSSARY(409, "ATLAS-400-00-0015", "Parent category from another Anchor(glossary) not supported"), CLASSIFICATION_TYPE_HAS_REFERENCES(409, "ATLAS-400-00-0016", "Given classification {0} [{1}] has references"), - // All internal errors go here INTERNAL_ERROR(500, "ATLAS-500-00-001", "Internal server error {0}"), INDEX_CREATION_FAILED(500, "ATLAS-500-00-002", "Index creation failed for {0}"), diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityGraphDiscoveryV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityGraphDiscoveryV2.java index e3bf08e80a..040822dccc 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityGraphDiscoveryV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityGraphDiscoveryV2.java @@ -357,6 +357,12 @@ void visitEntity(AtlasEntityType entityType, AtlasEntity entity) throws AtlasBas private void visitRelationships(AtlasEntityType entityType, AtlasEntity entity, List visitedAttributes) throws AtlasBaseException { for (String attrName : entityType.getRelationshipAttributes().keySet()) { + if (entity.hasRelationshipAttribute(attrName)){ + if (entity.hasAppendRelationshipAttribute(attrName) || entity.hasRemoveRelationshipAttribute(attrName)){ + throw new AtlasBaseException(AtlasErrorCode.ATTRIBUTE_ALREADY_EXISTS_IN_RELATIONSHIP_ATTRIBUTE, attrName); + } + } + // if attribute is not in 'relationshipAttributes', try 'attributes' // appendRelationshipAttribute will be ignored if same attribute is present // in relationshipAttribute diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AttributeMutationContext.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AttributeMutationContext.java index fa968c84bf..9bc1080971 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AttributeMutationContext.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AttributeMutationContext.java @@ -76,6 +76,11 @@ public AttributeMutationContext(EntityOperation op, AtlasVertex referringVertex, this.existingEdge = currentEdge; } + public AttributeMutationContext(EntityOperation op, AtlasVertex referringVertex, AtlasAttribute attribute, Object value, + String vertexProperty, AtlasType currentElementType) { + this(op, referringVertex, attribute, value, vertexProperty, currentElementType, null); + } + @Override public int hashCode() { return Objects.hash(op, referringVertex, attribute, value, vertexProperty, currentElementType, existingEdge); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 2858c31aab..03f9beb5b0 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -1072,8 +1072,6 @@ private void mapRelationshipAttributes(AtlasEntity entity, AtlasEntityType entit } } - updateModificationMetadata(vertex); - RequestContext.get().endMetricRecord(metric); } @@ -1118,8 +1116,6 @@ private void mapAppendRemoveRelationshipAttributes(AtlasEntity entity, AtlasEnti } } - updateModificationMetadata(vertex); - RequestContext.get().endMetricRecord(metric); if (LOG.isDebugEnabled()) { @@ -1710,7 +1706,7 @@ private AtlasEdge mapObjectIdValueUsingRelationship(AttributeMutationContext ctx return ret; } - private AtlasEdge getEdgeUsingRelationship(AttributeMutationContext ctx, EntityMutationContext context) throws AtlasBaseException { + private AtlasEdge getEdgeUsingRelationship(AttributeMutationContext ctx, EntityMutationContext context, boolean createEdge) throws AtlasBaseException { if (LOG.isDebugEnabled()) { LOG.debug("==> getEdgeUsingRelationship({})", ctx); } @@ -1767,14 +1763,23 @@ private AtlasEdge getEdgeUsingRelationship(AttributeMutationContext ctx, EntityM toVertex = attributeVertex; } - AtlasEdge edge = relationshipStore.getRelationship(fromVertex, toVertex, new AtlasRelationship(relationshipName)); + AtlasEdge edge = null; - if (edge != null && getStatus(edge) != DELETED) { - ret = edge; - } + if (createEdge) { + edge = relationshipStore.getOrCreate(fromVertex, toVertex, new AtlasRelationship(relationshipName)); + boolean isCreated = graphHelper.getCreatedTime(edge) == RequestContext.get().getRequestTime(); - RequestContext requestContext = RequestContext.get(); - requestContext.recordEntityUpdate(entityRetriever.toAtlasEntityHeader(toVertex)); + if (isCreated) { + // if relationship did not exist before and new relationship was created + // record entity update on both relationship vertices + recordEntityUpdateForNonRelationsipAttribute(fromVertex); + recordEntityUpdateForNonRelationsipAttribute(toVertex); + } + + } else { + edge = relationshipStore.getRelationship(fromVertex, toVertex, new AtlasRelationship(relationshipName)); + } + ret = edge; } } @@ -2022,27 +2027,6 @@ public List appendArrayValue(AttributeMutationContext ctx, EntityMutationContext AtlasAttribute inverseRefAttribute = attribute.getInverseRefAttribute(); Cardinality cardinality = attribute.getAttributeDef().getCardinality(); List newElementsCreated = new ArrayList<>(); - List currentElements; - - - if (isReference && !isSoftReference) { - // returns already attached assets - currentElements = (List) getCollectionElementsUsingRelationship(ctx.getReferringVertex(), attribute, isStructType); - } else { - currentElements = (List) getArrayElementsProperty(elementType, isSoftReference, ctx.getReferringVertex(), ctx.getVertexProperty()); - } - - if (PARTIAL_UPDATE.equals(ctx.getOp()) && attribute.getAttributeDef().isAppendOnPartialUpdate() && CollectionUtils.isNotEmpty(currentElements)) { - if (CollectionUtils.isEmpty(newElements)) { - newElements = new ArrayList<>(currentElements); - } else { - List mergedVal = new ArrayList<>(currentElements); - - mergedVal.addAll(newElements); - - newElements = mergedVal; - } - } boolean isNewElementsNull = newElements == null; @@ -2054,13 +2038,13 @@ public List appendArrayValue(AttributeMutationContext ctx, EntityMutationContext newElements = (List) newElements.stream().distinct().collect(Collectors.toList()); } + for (int index = 0; index < newElements.size(); index++) { - AtlasEdge existingEdge = (isSoftReference) ? null : getEdgeAt(currentElements, index, elementType); AttributeMutationContext arrCtx = new AttributeMutationContext(ctx.getOp(), ctx.getReferringVertex(), ctx.getAttribute(), newElements.get(index), - ctx.getVertexProperty(), elementType, existingEdge); + ctx.getVertexProperty(), elementType); - Object newEntry = mapCollectionElementsToVertex(arrCtx, context); + Object newEntry = getEdgeUsingRelationship(arrCtx, context, true); if (isReference && newEntry != null && newEntry instanceof AtlasEdge && inverseRefAttribute != null) { // Update the inverse reference value. @@ -2084,23 +2068,23 @@ public List appendArrayValue(AttributeMutationContext ctx, EntityMutationContext } if (isNewElementsNull) { - setArrayElementsProperty(elementType, isSoftReference, ctx.getReferringVertex(), ctx.getVertexProperty(), null, null, cardinality); + setArrayElementsProperty(elementType, isSoftReference, ctx.getReferringVertex(), ctx.getVertexProperty(), new ArrayList<>(0), new ArrayList<>(0), cardinality); } else { - setArrayElementsProperty(elementType, isSoftReference, ctx.getReferringVertex(), ctx.getVertexProperty(), newElements, currentElements, cardinality); + setArrayElementsProperty(elementType, isSoftReference, ctx.getReferringVertex(), ctx.getVertexProperty(), newElements, new ArrayList<>(0), cardinality); } switch (ctx.getAttribute().getRelationshipEdgeLabel()) { - case TERM_ASSIGNMENT_LABEL: addMeaningsToEntity(ctx, newElementsCreated, new ArrayList<>()); + case TERM_ASSIGNMENT_LABEL: addMeaningsToEntity(ctx, newElementsCreated, new ArrayList<>(0)); break; - case CATEGORY_TERMS_EDGE_LABEL: addCategoriesToTermEntity(ctx, newElementsCreated, new ArrayList<>()); + case CATEGORY_TERMS_EDGE_LABEL: addCategoriesToTermEntity(ctx, newElementsCreated, new ArrayList<>(0)); break; - case CATEGORY_PARENT_EDGE_LABEL: addCatParentAttr(ctx, newElementsCreated, new ArrayList<>()); + case CATEGORY_PARENT_EDGE_LABEL: addCatParentAttr(ctx, newElementsCreated, new ArrayList<>(0)); break; case PROCESS_INPUTS: - case PROCESS_OUTPUTS: addEdgesToContext(GraphHelper.getGuid(ctx.referringVertex), newElementsCreated, new ArrayList<>()); + case PROCESS_OUTPUTS: addEdgesToContext(GraphHelper.getGuid(ctx.referringVertex), newElementsCreated, new ArrayList<>(0)); break; } @@ -2139,29 +2123,35 @@ public List removeArrayValue(AttributeMutationContext ctx, EntityMutationContext for (int index = 0; index < elementsDeleted.size(); index++) { AttributeMutationContext arrCtx = new AttributeMutationContext(ctx.getOp(), ctx.getReferringVertex(), ctx.getAttribute(), elementsDeleted.get(index), - ctx.getVertexProperty(), elementType, null); + ctx.getVertexProperty(), elementType); + + Object deleteEntry = getEdgeUsingRelationship(arrCtx, context, false); - Object deleteEntry = getEdgeUsingRelationship(arrCtx, context); - if(deleteEntry != null) { - entityRelationsDeleted.add(deleteEntry); + // throw error if relation does not exist but requested to remove + if (deleteEntry == null) { + AtlasVertex attributeVertex = context.getDiscoveryContext().getResolvedEntityVertex(getGuid(arrCtx.getValue())); + throw new AtlasBaseException(AtlasErrorCode.RELATIONSHIP_DOES_NOT_EXIST, attribute.getRelationshipName(), + AtlasGraphUtilsV2.getIdFromVertex(attributeVertex), AtlasGraphUtilsV2.getIdFromVertex(ctx.getReferringVertex())); } + + entityRelationsDeleted.add(deleteEntry); } removedElements = removeArrayEntries(attribute, (List)entityRelationsDeleted, ctx); switch (ctx.getAttribute().getRelationshipEdgeLabel()) { - case TERM_ASSIGNMENT_LABEL: addMeaningsToEntity(ctx, new ArrayList<>() , removedElements); + case TERM_ASSIGNMENT_LABEL: addMeaningsToEntity(ctx, new ArrayList<>(0) , removedElements); break; - case CATEGORY_TERMS_EDGE_LABEL: addCategoriesToTermEntity(ctx, new ArrayList<>(), removedElements); + case CATEGORY_TERMS_EDGE_LABEL: addCategoriesToTermEntity(ctx, new ArrayList<>(0), removedElements); break; - case CATEGORY_PARENT_EDGE_LABEL: addCatParentAttr(ctx, new ArrayList<>(), removedElements); + case CATEGORY_PARENT_EDGE_LABEL: addCatParentAttr(ctx, new ArrayList<>(0), removedElements); break; case PROCESS_INPUTS: - case PROCESS_OUTPUTS: addEdgesToContext(GraphHelper.getGuid(ctx.referringVertex), new ArrayList<>(), removedElements); + case PROCESS_OUTPUTS: addEdgesToContext(GraphHelper.getGuid(ctx.referringVertex), new ArrayList<>(0), removedElements); break; } @@ -2894,12 +2884,15 @@ private List removeArrayEntries(AtlasAttribute attribute, List Date: Tue, 23 Jan 2024 16:12:54 +0530 Subject: [PATCH 2/2] Enhance error handling in append and remove support for bulk AP --- .../atlas/repository/store/graph/v2/EntityGraphMapper.java | 1 + 1 file changed, 1 insertion(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 03f9beb5b0..f624e30dc9 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -1071,6 +1071,7 @@ private void mapRelationshipAttributes(AtlasEntity entity, AtlasEntityType entit } } } + updateModificationMetadata(vertex); RequestContext.get().endMetricRecord(metric); }