From 64c29b17d1f1524e7f2eaeebb609620ba76347cf Mon Sep 17 00:00:00 2001 From: nikhilbonte21 Date: Wed, 8 May 2024 12:48:16 +0530 Subject: [PATCH 1/5] DG 1298 Form a QualifiedName for Domains and Products --- .../graph/v2/preprocessor/PreProcessorUtils.java | 2 ++ .../datamesh/AbstractDomainPreProcessor.java | 6 ++++++ .../datamesh/DataDomainPreProcessor.java | 12 ++++++++++++ .../datamesh/DataProductPreProcessor.java | 10 ++++++++++ 4 files changed, 30 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index c8728df251..56e291ae85 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -53,6 +53,8 @@ public class PreProcessorUtils { public static final String PARENT_DOMAIN_QN_ATTR = "parentDomainQualifiedName"; public static final String SUPER_DOMAIN_QN_ATTR = "superDomainQualifiedName"; + public static final String MIGRATION_CUSTOM_ATTRIBUTE = "isQualifiedNameMigrated"; + //Query models constants public static final String PREFIX_QUERY_QN = "default/collection/"; public static final String COLLECTION_QUALIFIED_NAME = "collectionQualifiedName"; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 0d2d57b61c..2d48e23787 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -65,6 +65,12 @@ public abstract class AbstractDomainPreProcessor implements PreProcessor { static final Set PARENT_ATTRIBUTES = new HashSet<>(Arrays.asList(SUPER_DOMAIN_QN_ATTR, PARENT_DOMAIN_QN_ATTR)); + static final Map customAttributes = new HashMap<>(); + + static { + customAttributes.put(MIGRATION_CUSTOM_ATTRIBUTE, "true"); + } + AbstractDomainPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, AtlasGraph graph) { this.entityRetriever = entityRetriever; this.typeRegistry = typeRegistry; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java index b30f6b3c3a..8fcc32c3dd 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java @@ -84,6 +84,7 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co private void processCreateDomain(AtlasEntity entity) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreateDomain"); String domainName = (String) entity.getAttribute(NAME); + String parentDomainQualifiedName = (String) entity.getAttribute(PARENT_DOMAIN_QN_ATTR); AtlasEntityHeader parentDomain = getParent(entity); @@ -91,6 +92,9 @@ private void processCreateDomain(AtlasEntity entity) throws AtlasBaseException { parentDomainQualifiedName = (String) parentDomain.getAttribute(QUALIFIED_NAME); } + entity.setAttribute(QUALIFIED_NAME, createQualifiedName(parentDomainQualifiedName)); + entity.setCustomAttributes(customAttributes); + domainExists(domainName, parentDomainQualifiedName); RequestContext.get().endMetricRecord(metricRecorder); @@ -307,6 +311,14 @@ private void domainExists(String domainName, String parentDomainQualifiedName) t RequestContext.get().endMetricRecord(metricRecorder); } } + + private static String createQualifiedName(String parentDomainQualifiedName) { + if (StringUtils.isNotEmpty(parentDomainQualifiedName)) { + return parentDomainQualifiedName + "/domain/" + getUUID(); + } else{ + return "default/domain/" + getUUID() + "/super"; + } + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 59048a498f..babebd7c90 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -65,6 +65,9 @@ private void processCreateProduct(AtlasEntity entity) throws AtlasBaseException parentDomainQualifiedName = (String) parentDomain.getAttribute(QUALIFIED_NAME); } + entity.setAttribute(QUALIFIED_NAME, createQualifiedName(parentDomainQualifiedName)); + entity.setCustomAttributes(customAttributes); + productExists(productName, parentDomainQualifiedName); RequestContext.get().endMetricRecord(metricRecorder); @@ -179,4 +182,11 @@ private void productExists(String productName, String parentDomainQualifiedName) } } + private static String createQualifiedName(String parentDomainQualifiedName) throws AtlasBaseException { + if (StringUtils.isEmpty(parentDomainQualifiedName)) { + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Parent Domain Qualified Name cannot be empty or null"); + } + return parentDomainQualifiedName + "/product/" + getUUID(); + + } } From f5cf2c3a22968ad13821fe499c9cbb0e3e9962cb Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte <89625355+nikhilbonte21@users.noreply.github.com> Date: Wed, 8 May 2024 22:02:12 +0530 Subject: [PATCH 2/5] Handle issue moving subDomain to make it Super Domain --- .../graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java | 1 + 1 file changed, 1 insertion(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java index 8fcc32c3dd..f2bec8a260 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java @@ -177,6 +177,7 @@ private void processMoveSubDomainToAnotherDomain(AtlasEntity domain, //Moving subDomain to make it Super Domain targetDomainQualifiedName = "default"; updatedQualifiedName = currentDomainQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); + updatedQualifiedName = updatedQualifiedName + "/super"; domain.setAttribute(QUALIFIED_NAME, updatedQualifiedName); domain.setAttribute(PARENT_DOMAIN_QN_ATTR, null); domain.setAttribute(SUPER_DOMAIN_QN_ATTR, null); From 95da7fd404f179e94e4f1c59f401ccce5442252e Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Thu, 9 May 2024 00:23:28 +0530 Subject: [PATCH 3/5] DG 1298 Handle issue moving subDomain to make it Super Domain --- .../datamesh/DataDomainPreProcessor.java | 30 +++++++++++++++++-- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java index f2bec8a260..ea09fb6947 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java @@ -190,7 +190,7 @@ private void processMoveSubDomainToAnotherDomain(AtlasEntity domain, domain.setAttribute(SUPER_DOMAIN_QN_ATTR, superDomainQualifiedName); } - moveChildrenToAnotherDomain(domainVertex, superDomainQualifiedName, null, sourceDomainQualifiedName, targetDomainQualifiedName); + moveChildren(domainVertex, superDomainQualifiedName, updatedQualifiedName, sourceDomainQualifiedName, targetDomainQualifiedName); updatePolicies(this.updatedPolicyResources, this.context); LOG.info("Moved subDomain {} to Domain {}", domainName, targetDomainQualifiedName); @@ -200,6 +200,25 @@ private void processMoveSubDomainToAnotherDomain(AtlasEntity domain, } } + private void moveChildren(AtlasVertex domainVertex, + String superDomainQualifiedName, + String parentDomainQualifiedName, + String sourceDomainQualifiedName, + String targetDomainQualifiedName) throws AtlasBaseException { + // move products to target Domain + Iterator products = getActiveChildrenVertices(domainVertex, DATA_PRODUCT_EDGE_LABEL); + while (products.hasNext()) { + AtlasVertex productVertex = products.next(); + moveChildDataProductToAnotherDomain(productVertex, superDomainQualifiedName, parentDomainQualifiedName, sourceDomainQualifiedName, targetDomainQualifiedName); + } + // Get all children domains of current domain + Iterator childDomains = getActiveChildrenVertices(domainVertex, DOMAIN_PARENT_EDGE_LABEL); + while (childDomains.hasNext()) { + AtlasVertex childVertex = childDomains.next(); + moveChildrenToAnotherDomain(childVertex, superDomainQualifiedName, parentDomainQualifiedName, sourceDomainQualifiedName, targetDomainQualifiedName); + } + } + private void moveChildrenToAnotherDomain(AtlasVertex childDomainVertex, String superDomainQualifiedName, String parentDomainQualifiedName, @@ -213,7 +232,7 @@ private void moveChildrenToAnotherDomain(AtlasVertex childDomainVertex, Map updatedAttributes = new HashMap<>(); String currentDomainQualifiedName = childDomainVertex.getProperty(QUALIFIED_NAME, String.class); - String updatedDomainQualifiedName = currentDomainQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); + String updatedDomainQualifiedName = parentDomainQualifiedName + getOwnQualifiedNameForChild(currentDomainQualifiedName); // Change domain qualifiedName childDomainVertex.setProperty(QUALIFIED_NAME, updatedDomainQualifiedName); @@ -269,7 +288,7 @@ private void moveChildDataProductToAnotherDomain(AtlasVertex productVertex, Map updatedAttributes = new HashMap<>(); String currentQualifiedName = productVertex.getProperty(QUALIFIED_NAME, String.class); - String updatedQualifiedName = currentQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); + String updatedQualifiedName = parentDomainQualifiedName + getOwnQualifiedNameForChild(currentQualifiedName); productVertex.setProperty(QUALIFIED_NAME, updatedQualifiedName); updatedAttributes.put(QUALIFIED_NAME, updatedQualifiedName); @@ -320,6 +339,11 @@ private static String createQualifiedName(String parentDomainQualifiedName) { return "default/domain/" + getUUID() + "/super"; } } + + private String getOwnQualifiedNameForChild(String childQualifiedName) { + String[] splitted = childQualifiedName.split("/"); + return String.format("/%s/%s", splitted[splitted.length -2], splitted[splitted.length -1]); + } } From 19217b21aff92713c69806cae7eb92aa0e580e09 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Thu, 9 May 2024 01:13:46 +0530 Subject: [PATCH 4/5] Logging --- .../preprocessor/datamesh/AbstractDomainPreProcessor.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 2d48e23787..796317a904 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -37,6 +37,7 @@ import org.apache.atlas.repository.store.graph.v2.preprocessor.AuthPolicyPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; import org.apache.atlas.type.AtlasEntityType; +import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; @@ -109,10 +110,15 @@ protected void updatePolicies(Map updatedPolicyResources, Entity try { AtlasEntityType entityType = typeRegistry.getEntityTypeByName(POLICY_ENTITY_TYPE); + LOG.info("updatedPolicyResources.size: {} ", updatedPolicyResources.size()); + LOG.info("updatedPolicyResources {} ", AtlasType.toJson(updatedPolicyResources)); + List policies = getPolicies(updatedPolicyResources.keySet()); + LOG.info("policies.size: {} ", policies.size()); if (CollectionUtils.isNotEmpty(policies)) { for (AtlasEntityHeader policy : policies) { + LOG.info("Updating policy {} ", policy.getGuid()); AtlasVertex policyVertex = entityRetriever.getEntityVertex(policy.getGuid()); AtlasEntity policyEntity = entityRetriever.toAtlasEntity(policyVertex); From dbdccec5a0eb3c0e823c738e8978bc9badfeaa6d Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Thu, 9 May 2024 01:31:15 +0530 Subject: [PATCH 5/5] Fix policy not updating issue --- .../datamesh/AbstractDomainPreProcessor.java | 11 ++++++----- .../preprocessor/datamesh/DataDomainPreProcessor.java | 5 ++++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 796317a904..936ba6d948 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -37,10 +37,10 @@ import org.apache.atlas.repository.store.graph.v2.preprocessor.AuthPolicyPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; import org.apache.atlas.type.AtlasEntityType; -import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -110,15 +110,16 @@ protected void updatePolicies(Map updatedPolicyResources, Entity try { AtlasEntityType entityType = typeRegistry.getEntityTypeByName(POLICY_ENTITY_TYPE); - LOG.info("updatedPolicyResources.size: {} ", updatedPolicyResources.size()); - LOG.info("updatedPolicyResources {} ", AtlasType.toJson(updatedPolicyResources)); + if (MapUtils.isEmpty(updatedPolicyResources)) { + return; + } List policies = getPolicies(updatedPolicyResources.keySet()); - LOG.info("policies.size: {} ", policies.size()); + LOG.info("Found {} policies to update", policies.size()); if (CollectionUtils.isNotEmpty(policies)) { for (AtlasEntityHeader policy : policies) { - LOG.info("Updating policy {} ", policy.getGuid()); + LOG.info("Updating Policy {}", policy.getGuid()); AtlasVertex policyVertex = entityRetriever.getEntityVertex(policy.getGuid()); AtlasEntity policyEntity = entityRetriever.toAtlasEntity(policyVertex); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java index ea09fb6947..f17a36e833 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java @@ -190,6 +190,9 @@ private void processMoveSubDomainToAnotherDomain(AtlasEntity domain, domain.setAttribute(SUPER_DOMAIN_QN_ATTR, superDomainQualifiedName); } + String currentQualifiedName = domainVertex.getProperty(QUALIFIED_NAME, String.class); + this.updatedPolicyResources.put("entity:" + currentQualifiedName, "entity:" + updatedQualifiedName); + moveChildren(domainVertex, superDomainQualifiedName, updatedQualifiedName, sourceDomainQualifiedName, targetDomainQualifiedName); updatePolicies(this.updatedPolicyResources, this.context); @@ -339,7 +342,7 @@ private static String createQualifiedName(String parentDomainQualifiedName) { return "default/domain/" + getUUID() + "/super"; } } - + private String getOwnQualifiedNameForChild(String childQualifiedName) { String[] splitted = childQualifiedName.split("/"); return String.format("/%s/%s", splitted[splitted.length -2], splitted[splitted.length -1]);