diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index c8728df251..56e291ae85 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -53,6 +53,8 @@ public class PreProcessorUtils { public static final String PARENT_DOMAIN_QN_ATTR = "parentDomainQualifiedName"; public static final String SUPER_DOMAIN_QN_ATTR = "superDomainQualifiedName"; + public static final String MIGRATION_CUSTOM_ATTRIBUTE = "isQualifiedNameMigrated"; + //Query models constants public static final String PREFIX_QUERY_QN = "default/collection/"; public static final String COLLECTION_QUALIFIED_NAME = "collectionQualifiedName"; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 0d2d57b61c..936ba6d948 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -40,6 +40,7 @@ import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -65,6 +66,12 @@ public abstract class AbstractDomainPreProcessor implements PreProcessor { static final Set PARENT_ATTRIBUTES = new HashSet<>(Arrays.asList(SUPER_DOMAIN_QN_ATTR, PARENT_DOMAIN_QN_ATTR)); + static final Map customAttributes = new HashMap<>(); + + static { + customAttributes.put(MIGRATION_CUSTOM_ATTRIBUTE, "true"); + } + AbstractDomainPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, AtlasGraph graph) { this.entityRetriever = entityRetriever; this.typeRegistry = typeRegistry; @@ -103,10 +110,16 @@ protected void updatePolicies(Map updatedPolicyResources, Entity try { AtlasEntityType entityType = typeRegistry.getEntityTypeByName(POLICY_ENTITY_TYPE); + if (MapUtils.isEmpty(updatedPolicyResources)) { + return; + } + List policies = getPolicies(updatedPolicyResources.keySet()); + LOG.info("Found {} policies to update", policies.size()); if (CollectionUtils.isNotEmpty(policies)) { for (AtlasEntityHeader policy : policies) { + LOG.info("Updating Policy {}", policy.getGuid()); AtlasVertex policyVertex = entityRetriever.getEntityVertex(policy.getGuid()); AtlasEntity policyEntity = entityRetriever.toAtlasEntity(policyVertex); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java index b30f6b3c3a..f17a36e833 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java @@ -84,6 +84,7 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co private void processCreateDomain(AtlasEntity entity) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreateDomain"); String domainName = (String) entity.getAttribute(NAME); + String parentDomainQualifiedName = (String) entity.getAttribute(PARENT_DOMAIN_QN_ATTR); AtlasEntityHeader parentDomain = getParent(entity); @@ -91,6 +92,9 @@ private void processCreateDomain(AtlasEntity entity) throws AtlasBaseException { parentDomainQualifiedName = (String) parentDomain.getAttribute(QUALIFIED_NAME); } + entity.setAttribute(QUALIFIED_NAME, createQualifiedName(parentDomainQualifiedName)); + entity.setCustomAttributes(customAttributes); + domainExists(domainName, parentDomainQualifiedName); RequestContext.get().endMetricRecord(metricRecorder); @@ -173,6 +177,7 @@ private void processMoveSubDomainToAnotherDomain(AtlasEntity domain, //Moving subDomain to make it Super Domain targetDomainQualifiedName = "default"; updatedQualifiedName = currentDomainQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); + updatedQualifiedName = updatedQualifiedName + "/super"; domain.setAttribute(QUALIFIED_NAME, updatedQualifiedName); domain.setAttribute(PARENT_DOMAIN_QN_ATTR, null); domain.setAttribute(SUPER_DOMAIN_QN_ATTR, null); @@ -185,7 +190,10 @@ private void processMoveSubDomainToAnotherDomain(AtlasEntity domain, domain.setAttribute(SUPER_DOMAIN_QN_ATTR, superDomainQualifiedName); } - moveChildrenToAnotherDomain(domainVertex, superDomainQualifiedName, null, sourceDomainQualifiedName, targetDomainQualifiedName); + String currentQualifiedName = domainVertex.getProperty(QUALIFIED_NAME, String.class); + this.updatedPolicyResources.put("entity:" + currentQualifiedName, "entity:" + updatedQualifiedName); + + moveChildren(domainVertex, superDomainQualifiedName, updatedQualifiedName, sourceDomainQualifiedName, targetDomainQualifiedName); updatePolicies(this.updatedPolicyResources, this.context); LOG.info("Moved subDomain {} to Domain {}", domainName, targetDomainQualifiedName); @@ -195,6 +203,25 @@ private void processMoveSubDomainToAnotherDomain(AtlasEntity domain, } } + private void moveChildren(AtlasVertex domainVertex, + String superDomainQualifiedName, + String parentDomainQualifiedName, + String sourceDomainQualifiedName, + String targetDomainQualifiedName) throws AtlasBaseException { + // move products to target Domain + Iterator products = getActiveChildrenVertices(domainVertex, DATA_PRODUCT_EDGE_LABEL); + while (products.hasNext()) { + AtlasVertex productVertex = products.next(); + moveChildDataProductToAnotherDomain(productVertex, superDomainQualifiedName, parentDomainQualifiedName, sourceDomainQualifiedName, targetDomainQualifiedName); + } + // Get all children domains of current domain + Iterator childDomains = getActiveChildrenVertices(domainVertex, DOMAIN_PARENT_EDGE_LABEL); + while (childDomains.hasNext()) { + AtlasVertex childVertex = childDomains.next(); + moveChildrenToAnotherDomain(childVertex, superDomainQualifiedName, parentDomainQualifiedName, sourceDomainQualifiedName, targetDomainQualifiedName); + } + } + private void moveChildrenToAnotherDomain(AtlasVertex childDomainVertex, String superDomainQualifiedName, String parentDomainQualifiedName, @@ -208,7 +235,7 @@ private void moveChildrenToAnotherDomain(AtlasVertex childDomainVertex, Map updatedAttributes = new HashMap<>(); String currentDomainQualifiedName = childDomainVertex.getProperty(QUALIFIED_NAME, String.class); - String updatedDomainQualifiedName = currentDomainQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); + String updatedDomainQualifiedName = parentDomainQualifiedName + getOwnQualifiedNameForChild(currentDomainQualifiedName); // Change domain qualifiedName childDomainVertex.setProperty(QUALIFIED_NAME, updatedDomainQualifiedName); @@ -264,7 +291,7 @@ private void moveChildDataProductToAnotherDomain(AtlasVertex productVertex, Map updatedAttributes = new HashMap<>(); String currentQualifiedName = productVertex.getProperty(QUALIFIED_NAME, String.class); - String updatedQualifiedName = currentQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); + String updatedQualifiedName = parentDomainQualifiedName + getOwnQualifiedNameForChild(currentQualifiedName); productVertex.setProperty(QUALIFIED_NAME, updatedQualifiedName); updatedAttributes.put(QUALIFIED_NAME, updatedQualifiedName); @@ -307,6 +334,19 @@ private void domainExists(String domainName, String parentDomainQualifiedName) t RequestContext.get().endMetricRecord(metricRecorder); } } + + private static String createQualifiedName(String parentDomainQualifiedName) { + if (StringUtils.isNotEmpty(parentDomainQualifiedName)) { + return parentDomainQualifiedName + "/domain/" + getUUID(); + } else{ + return "default/domain/" + getUUID() + "/super"; + } + } + + private String getOwnQualifiedNameForChild(String childQualifiedName) { + String[] splitted = childQualifiedName.split("/"); + return String.format("/%s/%s", splitted[splitted.length -2], splitted[splitted.length -1]); + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 59048a498f..babebd7c90 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -65,6 +65,9 @@ private void processCreateProduct(AtlasEntity entity) throws AtlasBaseException parentDomainQualifiedName = (String) parentDomain.getAttribute(QUALIFIED_NAME); } + entity.setAttribute(QUALIFIED_NAME, createQualifiedName(parentDomainQualifiedName)); + entity.setCustomAttributes(customAttributes); + productExists(productName, parentDomainQualifiedName); RequestContext.get().endMetricRecord(metricRecorder); @@ -179,4 +182,11 @@ private void productExists(String productName, String parentDomainQualifiedName) } } + private static String createQualifiedName(String parentDomainQualifiedName) throws AtlasBaseException { + if (StringUtils.isEmpty(parentDomainQualifiedName)) { + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Parent Domain Qualified Name cannot be empty or null"); + } + return parentDomainQualifiedName + "/product/" + getUUID(); + + } }