From 03da5ca9ba9732ca4fbc8666d7929a9bac9f85aa Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Wed, 21 Aug 2024 20:38:41 +0530 Subject: [PATCH] feat: add qualifiedName hierarchy after preprocessor --- .../store/aliasstore/ESAliasStore.java | 6 +- .../store/graph/v2/AtlasEntityStoreV2.java | 63 ++++++++++++------- 2 files changed, 45 insertions(+), 24 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/aliasstore/ESAliasStore.java b/repository/src/main/java/org/apache/atlas/repository/store/aliasstore/ESAliasStore.java index 499b572b76..b5962de2b7 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/aliasstore/ESAliasStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/aliasstore/ESAliasStore.java @@ -210,8 +210,12 @@ private void personaPolicyToESDslClauses(List policies, * This will be dictated by the feature flag ENABLE_PERSONA_HIERARCHY_FILTER */ + // If asset resource ends with /* then add it in hierarchical filter + boolean isHierarchical = asset.endsWith("/*"); + if (isHierarchical) { + asset = asset.substring(0, asset.length() - 2); + } boolean isWildcard = asset.contains("*") || asset.contains("?"); - if (isWildcard) { allowClauseList.add(mapOf("wildcard", mapOf(QUALIFIED_NAME, asset))); } else if (useHierarchicalQualifiedNameFilter) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 80401353aa..eb66158603 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -1456,7 +1456,6 @@ private EntityMutationResponse createOrUpdate(EntityStream entityStream, boolean AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, new AtlasEntityHeader(entity)), "create entity: type=", entity.getTypeName()); } - createQualifiedNameHierarchyField(entity, context.getVertex(entity.getGuid())); } } // for existing entities, skip update if incoming entity doesn't have any change @@ -1476,10 +1475,6 @@ private EntityMutationResponse createOrUpdate(EntityStream entityStream, boolean AtlasEntityDiffResult diffResult = entityComparator.getDiffResult(entity, storedVertex, !storeDifferentialAudits); if (diffResult.hasDifference()) { - if (diffResult.getDiffEntity().hasAttribute(QUALIFIED_NAME)) { - createQualifiedNameHierarchyField(entity, storedVertex); - } - if (storeDifferentialAudits) { diffResult.getDiffEntity().setGuid(entity.getGuid()); reqContext.cacheDifferentialEntity(diffResult.getDiffEntity()); @@ -1534,6 +1529,20 @@ private EntityMutationResponse createOrUpdate(EntityStream entityStream, boolean executePreProcessor(context); + // Updating hierarchy after preprocessor is executed so that qualifiedName update during preprocessor is considered + for (AtlasEntity entity : context.getCreatedEntities()) { + createQualifiedNameHierarchyField(entity, context.getVertex(entity.getGuid())); + } + + for (AtlasEntity entity : context.getUpdatedEntities()) { + // If qualifiedName update is part of the update, update the qualifiedName hierarchy field + AtlasEntity diffEntity = RequestContext.get().getDifferentialEntitiesMap().get(entity.getGuid()); + if (diffEntity != null && diffEntity.hasAttribute(QUALIFIED_NAME)) { + createQualifiedNameHierarchyField(entity, context.getVertex(entity.getGuid())); + } + } + + EntityMutationResponse ret = entityGraphMapper.mapAttributesAndClassifications(context, isPartialUpdate, replaceClassifications, replaceBusinessAttributes, isOverwriteBusinessAttribute); @@ -1808,28 +1817,36 @@ private AtlasStruct getStarredDetailsStruct(String assetStarredBy, long assetSta } private void createQualifiedNameHierarchyField(AtlasEntity entity, AtlasVertex vertex) { - if (entity.hasAttribute(QUALIFIED_NAME)) { - String qualifiedName = (String) entity.getAttribute(QUALIFIED_NAME); - if (StringUtils.isNotEmpty(qualifiedName)) { - vertex.removeProperty(QUALIFIED_NAME_HIERARCHY_PROPERTY_KEY); - String[] parts = qualifiedName.split("/"); - StringBuilder currentPath = new StringBuilder(); - - for (int i = 0; i < parts.length; i++) { - String part = parts[i]; - if (StringUtils.isNotEmpty(part)) { - if (i > 0) { - currentPath.append("/"); - } - currentPath.append(part); - // i>1 reason: we don't want to add the first part of the qualifiedName as it is the entity name - // Example qualifiedName : default/snowflake/123/db_name we only want `default/snowflake/123` and `default/snowflake/123/db_name` - if (i > 1) { - AtlasGraphUtilsV2.addEncodedProperty(vertex, QUALIFIED_NAME_HIERARCHY_PROPERTY_KEY, currentPath.toString()); + MetricRecorder metric = RequestContext.get().startMetricRecord("createQualifiedNameHierarchyField"); + try { + if (vertex == null) { + vertex = AtlasGraphUtilsV2.findByGuid(graph, entity.getGuid()); + } + if (entity.hasAttribute(QUALIFIED_NAME)) { + String qualifiedName = (String) entity.getAttribute(QUALIFIED_NAME); + if (StringUtils.isNotEmpty(qualifiedName)) { + vertex.removeProperty(QUALIFIED_NAME_HIERARCHY_PROPERTY_KEY); + String[] parts = qualifiedName.split("/"); + StringBuilder currentPath = new StringBuilder(); + + for (int i = 0; i < parts.length; i++) { + String part = parts[i]; + if (StringUtils.isNotEmpty(part)) { + if (i > 0) { + currentPath.append("/"); + } + currentPath.append(part); + // i>1 reason: we don't want to add the first part of the qualifiedName as it is the entity name + // Example qualifiedName : default/snowflake/123/db_name we only want `default/snowflake/123` and `default/snowflake/123/db_name` + if (i > 1) { + AtlasGraphUtilsV2.addEncodedProperty(vertex, QUALIFIED_NAME_HIERARCHY_PROPERTY_KEY, currentPath.toString()); + } } } } } + } finally { + RequestContext.get().endMetricRecord(metric); } }