From c48284d3c71426a920b1c23179d4b54e83ed8893 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Mon, 11 Mar 2024 23:50:17 +0530 Subject: [PATCH 001/316] Pre Processors for Domain Move --- .../apache/atlas/repository/Constants.java | 9 + .../store/graph/v2/AtlasEntityStoreV2.java | 10 + .../v2/preprocessor/PreProcessorUtils.java | 6 + .../datamesh/AbstractDomainPreProcessor.java | 159 ++++++++++ .../datamesh/DataProductPreProcessor.java | 186 +++++++++++ .../datamesh/DomainPreProcessor.java | 299 ++++++++++++++++++ 6 files changed, 669 insertions(+) create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 4a2cd5905c7..37b6d7416b8 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -134,6 +134,15 @@ public final class Constants { public static final String GLOSSARY_TERMS_EDGE_LABEL = "r:AtlasGlossaryTermAnchor"; public static final String GLOSSARY_CATEGORY_EDGE_LABEL = "r:AtlasGlossaryCategoryAnchor"; + /** + * MESH property keys. + */ + public static final String DATA_DOMAIN_ENTITY_TYPE = "DataDomain"; + public static final String DATA_PRODUCT_ENTITY_TYPE = "DataProduct"; + public static final String DATA_PRODUCT_EDGE_LABEL = "r:AtlasDataProduct"; + public static final String DOMAIN_PARENT_EDGE_LABEL = "r:AtlasDataDomainHierarchyLink"; + + /** * SQL property keys. diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 6e3487f8d37..472c29cc468 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -62,6 +62,8 @@ import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.accesscontrol.PersonaPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.accesscontrol.PurposePreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.datamesh.DataProductPreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.datamesh.DomainPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.glossary.CategoryPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.glossary.GlossaryPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.glossary.TermPreProcessor; @@ -1800,6 +1802,14 @@ public PreProcessor getPreProcessor(String typeName) { preProcessor = new CategoryPreProcessor(typeRegistry, entityRetriever, graph, taskManagement, entityGraphMapper); break; + case DATA_DOMAIN_ENTITY_TYPE: + preProcessor = new DomainPreProcessor(typeRegistry, entityRetriever, graph, entityGraphMapper); + break; + + case DATA_PRODUCT_ENTITY_TYPE: + preProcessor = new DataProductPreProcessor(typeRegistry, entityRetriever, graph, entityGraphMapper); + break; + case QUERY_ENTITY_TYPE: preProcessor = new QueryPreProcessor(typeRegistry, entityRetriever); break; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index 6c849004602..b23ab7c3f9b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -32,6 +32,12 @@ public class PreProcessorUtils { public static final String GLOSSARY_TERM_REL_TYPE = "AtlasGlossaryTermAnchor"; public static final String GLOSSARY_CATEGORY_REL_TYPE = "AtlasGlossaryCategoryAnchor"; + //DataMesh models constants + public static final String DATA_PRODUCT_TYPE = "DataProduct"; + public static final String PARENT_DOMAIN = "parentDomain"; + public static final String PARENT_DOMAIN_QN = "parentDomainQualifiedName"; + public static final String SUPER_DOMAIN_QN = "superDomainQualifiedName"; + //Query models constants public static final String PREFIX_QUERY_QN = "default/collection/"; public static final String COLLECTION_QUALIFIED_NAME = "collectionQualifiedName"; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java new file mode 100644 index 00000000000..5f903caeb5e --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -0,0 +1,159 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.store.graph.v2.preprocessor.datamesh; + +import org.apache.atlas.AtlasException; +import org.apache.atlas.RequestContext; +import org.apache.atlas.authorize.AtlasAuthorizationUtils; +import org.apache.atlas.authorize.AtlasEntityAccessRequest; +import org.apache.atlas.authorize.AtlasPrivilege; +import org.apache.atlas.discovery.EntityDiscoveryService; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.discovery.IndexSearchParams; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.glossary.AbstractGlossaryPreProcessor; +import org.apache.atlas.type.AtlasEntityType; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.apache.commons.collections.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +import static org.apache.atlas.repository.Constants.NAME; +import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; + +public abstract class AbstractDomainPreProcessor implements PreProcessor { + private static final Logger LOG = LoggerFactory.getLogger(AbstractGlossaryPreProcessor.class); + + + protected final AtlasTypeRegistry typeRegistry; + protected final EntityGraphRetriever entityRetriever; + + protected EntityDiscoveryService discovery; + + AbstractDomainPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, AtlasGraph graph) { + this.entityRetriever = entityRetriever; + this.typeRegistry = typeRegistry; + + try { + this.discovery = new EntityDiscoveryService(typeRegistry, graph, null, null, null, null); + } catch (AtlasException e) { + e.printStackTrace(); + } + } + + public List indexSearchPaginated(Map dsl) throws AtlasBaseException { + IndexSearchParams searchParams = new IndexSearchParams(); + List ret = new ArrayList<>(); + + List sortList = new ArrayList<>(0); + sortList.add(mapOf("__timestamp", mapOf("order", "asc"))); + sortList.add(mapOf("__guid", mapOf("order", "asc"))); + dsl.put("sort", sortList); + + int from = 0; + int size = 100; + boolean hasMore = true; + do { + dsl.put("from", from); + dsl.put("size", size); + searchParams.setDsl(dsl); + + List headers = discovery.directIndexSearch(searchParams).getEntities(); + + if (CollectionUtils.isNotEmpty(headers)) { + ret.addAll(headers); + } else { + hasMore = false; + } + + from += size; + + } while (hasMore); + + return ret; + } + + protected void isAuthorized(AtlasEntityHeader sourceDomain, AtlasEntityHeader targetDomain) throws AtlasBaseException { + + // source -> CREATE + UPDATE + DELETE + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, sourceDomain), + "create on source Domain: ", sourceDomain.getAttribute(NAME)); + + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, sourceDomain), + "update on source Domain: ", sourceDomain.getAttribute(NAME)); + + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_DELETE, sourceDomain), + "delete on source Domain: ", sourceDomain.getAttribute(NAME)); + + + // target -> CREATE + UPDATE + DELETE + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, targetDomain), + "create on target Domain: ", targetDomain.getAttribute(NAME)); + + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, targetDomain), + "update on target Glossary: ", targetDomain.getAttribute(NAME)); + + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_DELETE, targetDomain), + "delete on target Glossary: ", targetDomain.getAttribute(NAME)); + } + + /** + * Record the updated child entities, it will be used to send notification and store audit logs + * @param entityVertex Child entity vertex + * @param updatedAttributes Updated attributes while updating required attributes on updating collection + */ + protected void recordUpdatedChildEntities(AtlasVertex entityVertex, Map updatedAttributes) { + RequestContext requestContext = RequestContext.get(); + AtlasPerfMetrics.MetricRecorder metricRecorder = requestContext.startMetricRecord("recordUpdatedChildEntities"); + AtlasEntity entity = new AtlasEntity(); + entity = entityRetriever.mapSystemAttributes(entityVertex, entity); + entity.setAttributes(updatedAttributes); + requestContext.cacheDifferentialEntity(new AtlasEntity(entity)); + + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName()); + + //Add the min info attributes to entity header to be sent as part of notification + if(entityType != null) { + AtlasEntity finalEntity = entity; + entityType.getMinInfoAttributes().values().stream().filter(attribute -> !updatedAttributes.containsKey(attribute.getName())).forEach(attribute -> { + Object attrValue = null; + try { + attrValue = entityRetriever.getVertexAttribute(entityVertex, attribute); + } catch (AtlasBaseException e) { + LOG.error("Error while getting vertex attribute", e); + } + if(attrValue != null) { + finalEntity.setAttribute(attribute.getName(), attrValue); + } + }); + requestContext.recordEntityUpdate(new AtlasEntityHeader(finalEntity)); + } + + requestContext.endMetricRecord(metricRecorder); + } +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java new file mode 100644 index 00000000000..b3f596439cb --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -0,0 +1,186 @@ +package org.apache.atlas.repository.store.graph.v2.preprocessor.datamesh; + +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.RequestContext; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.*; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; +import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.atlas.repository.Constants.*; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.PARENT_DOMAIN; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.PARENT_DOMAIN_QN; +import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; + +public class DataProductPreProcessor extends AbstractDomainPreProcessor { + private static final Logger LOG = LoggerFactory.getLogger(DomainPreProcessor.class); + private AtlasEntityHeader parentDomain; + public DataProductPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, + AtlasGraph graph, EntityGraphMapper entityGraphMapper) { + super(typeRegistry, entityRetriever, graph); + } + + @Override + public void processAttributes(AtlasStruct entityStruct, EntityMutationContext context, + EntityMutations.EntityOperation operation) throws AtlasBaseException { + //Handle name & qualifiedName + if (LOG.isDebugEnabled()) { + LOG.debug("DataProductPreProcessor.processAttributes: pre processing {}, {}", + entityStruct.getAttribute(QUALIFIED_NAME), operation); + } + +// this.context = context; + + AtlasEntity entity = (AtlasEntity) entityStruct; + AtlasVertex vertex = context.getVertex(entity.getGuid()); + + setParent(entity, context); + processUpdateDomain(entity, vertex); + } + + private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdateDomain"); + String productName = (String) entity.getAttribute(NAME); + String vertexQnName = vertex.getProperty(QUALIFIED_NAME, String.class); + +// if (StringUtils.isEmpty(catName) || isNameInvalid(catName)) { +// throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); +// } + + AtlasEntity storedProduct = entityRetriever.toAtlasEntity(vertex); + AtlasRelatedObjectId currentDomain = (AtlasRelatedObjectId) storedProduct.getRelationshipAttribute(PARENT_DOMAIN); + AtlasEntityHeader currentDomainHeader = entityRetriever.toAtlasEntityHeader(currentDomain.getGuid()); + String currentDomainQualifiedName = (String) currentDomainHeader.getAttribute(QUALIFIED_NAME); + + String newDomainQualifiedName = (String) parentDomain.getAttribute(QUALIFIED_NAME); + + if (!currentDomainQualifiedName.equals(newDomainQualifiedName)) { + //Auth check + isAuthorized(currentDomainHeader, parentDomain); + + processMoveDataProductToAnotherDomain(entity, currentDomainQualifiedName, newDomainQualifiedName, vertexQnName); + entity.setAttribute(PARENT_DOMAIN_QN, newDomainQualifiedName); + + } else { + String vertexName = vertex.getProperty(NAME, String.class); + if (!vertexName.equals(productName)) { + domainExists(productName, newDomainQualifiedName); + } + + entity.setAttribute(QUALIFIED_NAME, vertexQnName); + } + + RequestContext.get().endMetricRecord(metricRecorder); + } + + private void processMoveDataProductToAnotherDomain(AtlasEntity product, + String sourceDomainQualifiedName, + String targetDomainQualifiedName, + String currentDataProductQualifiedName) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("processMoveDataProductToAnotherDomain"); + + try { + String domainName = (String) product.getAttribute(NAME); + + LOG.info("Moving dataProduct {} to Domain {}", domainName, targetDomainQualifiedName); + + domainExists(domainName, targetDomainQualifiedName); + + String updatedQualifiedName = currentDataProductQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); + + product.setAttribute(QUALIFIED_NAME, updatedQualifiedName); + + LOG.info("Moved dataProduct {} to Domain {}", domainName, targetDomainQualifiedName); + + } finally { + RequestContext.get().endMetricRecord(recorder); + } + } + + private void setParent(AtlasEntity entity, EntityMutationContext context) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("DataProductPreProcessor.setParent"); + if (parentDomain == null) { + AtlasObjectId objectId = (AtlasObjectId) entity.getRelationshipAttribute(PARENT_DOMAIN); + + if (objectId != null) { + if (StringUtils.isNotEmpty(objectId.getGuid())) { + AtlasVertex vertex = context.getVertex(objectId.getGuid()); + + if (vertex == null) { + parentDomain = entityRetriever.toAtlasEntityHeader(objectId.getGuid()); + } else { + parentDomain = entityRetriever.toAtlasEntityHeader(vertex); + } + + } else if (MapUtils.isNotEmpty(objectId.getUniqueAttributes()) && + StringUtils.isNotEmpty((String) objectId.getUniqueAttributes().get(QUALIFIED_NAME))) { + parentDomain = new AtlasEntityHeader(objectId.getTypeName(), objectId.getUniqueAttributes()); + + } + } + } + RequestContext.get().endMetricRecord(metricRecorder); + } + + private void domainExists(String domainName, String domainQualifiedName) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("domainExists"); + + boolean exists = false; + try { + List mustClauseList = new ArrayList(); + mustClauseList.add(mapOf("term", mapOf("__dataDomain", domainQualifiedName))); + mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", DATA_DOMAIN_ENTITY_TYPE))); + mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); + mustClauseList.add(mapOf("term", mapOf("name.keyword", domainName))); + + + Map bool = new HashMap<>(); + if (parentDomain != null) { + String parentQname = (String) parentDomain.getAttribute(QUALIFIED_NAME); + mustClauseList.add(mapOf("term", mapOf("__parentDomain", parentQname))); + } else { + List mustNotClauseList = new ArrayList(); + mustNotClauseList.add(mapOf("exists", mapOf("field", "__parentDomain"))); + bool.put("must_not", mustNotClauseList); + } + + bool.put("must", mustClauseList); + + Map dsl = mapOf("query", mapOf("bool", bool)); + + List domains = indexSearchPaginated(dsl); + + if (CollectionUtils.isNotEmpty(domains)) { + for (AtlasEntityHeader domain : domains) { + String name = (String) domain.getAttribute(NAME); + if (domainName.equals(name)) { + exists = true; + break; + } + } + } + } finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + + if (exists) { + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, domainName); + } + } + +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java new file mode 100644 index 00000000000..fd64c719f5d --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -0,0 +1,299 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.store.graph.v2.preprocessor.datamesh; + + +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.RequestContext; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.model.instance.AtlasObjectId; +import org.apache.atlas.model.instance.AtlasRelatedObjectId; +import org.apache.atlas.model.instance.AtlasStruct; +import org.apache.atlas.model.instance.EntityMutations; +import org.apache.atlas.repository.graph.GraphHelper; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; +import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import static org.apache.atlas.repository.Constants.*; +import static org.apache.atlas.repository.graph.GraphHelper.getActiveChildrenVertices; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.*; +import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; + +public class DomainPreProcessor extends AbstractDomainPreProcessor { + private static final Logger LOG = LoggerFactory.getLogger(DomainPreProcessor.class); + private AtlasEntityHeader parentDomain; + private EntityGraphMapper entityGraphMapper; + private EntityMutationContext context; + + public DomainPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, + AtlasGraph graph, EntityGraphMapper entityGraphMapper) { + super(typeRegistry, entityRetriever, graph); + this.entityGraphMapper = entityGraphMapper; + } + + @Override + public void processAttributes(AtlasStruct entityStruct, EntityMutationContext context, + EntityMutations.EntityOperation operation) throws AtlasBaseException { + //Handle name & qualifiedName + if (LOG.isDebugEnabled()) { + LOG.debug("DomainPreProcessor.processAttributes: pre processing {}, {}", + entityStruct.getAttribute(QUALIFIED_NAME), operation); + } + + this.context = context; + + AtlasEntity entity = (AtlasEntity) entityStruct; + AtlasVertex vertex = context.getVertex(entity.getGuid()); + + setParent(entity, context); + processUpdateDomain(entity, vertex); + } + + private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdateDomain"); + String domainName = (String) entity.getAttribute(NAME); + String vertexQnName = vertex.getProperty(QUALIFIED_NAME, String.class); + + AtlasEntity storedDomain = entityRetriever.toAtlasEntity(vertex); + if(storedDomain.getRelationshipAttribute(PARENT_DOMAIN) == null){ + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Cannot move Root Domain"); + } + AtlasRelatedObjectId currentDomain = (AtlasRelatedObjectId) storedDomain.getRelationshipAttribute(PARENT_DOMAIN); + AtlasEntityHeader currentDomainHeader = entityRetriever.toAtlasEntityHeader(currentDomain.getGuid()); + String currentDomainQualifiedName = (String) currentDomainHeader.getAttribute(QUALIFIED_NAME); + + String newDomainQualifiedName = (String) parentDomain.getAttribute(QUALIFIED_NAME); + + + if (!currentDomainQualifiedName.equals(newDomainQualifiedName)) { + //Auth check + isAuthorized(currentDomainHeader, parentDomain); + + processMoveSubDomainToAnotherDomain(entity, vertex, currentDomainQualifiedName, newDomainQualifiedName, vertexQnName); + entity.setAttribute(PARENT_DOMAIN_QN, newDomainQualifiedName); + + } else { + String vertexName = vertex.getProperty(NAME, String.class); + if (!vertexName.equals(domainName)) { + domainExists(domainName, newDomainQualifiedName); + } + + entity.setAttribute(QUALIFIED_NAME, vertexQnName); + } + + RequestContext.get().endMetricRecord(metricRecorder); + } + + private void processMoveSubDomainToAnotherDomain(AtlasEntity domain, + AtlasVertex domainVertex, + String sourceDomainQualifiedName, + String targetDomainQualifiedName, + String currentSubDomainQualifiedName) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("processMoveSubDomainToAnotherGlossary"); + + try { + String domainName = (String) domain.getAttribute(NAME); + + LOG.info("Moving subdomain {} to Domain {}", domainName, targetDomainQualifiedName); + + domainExists(domainName, targetDomainQualifiedName); + + String updatedQualifiedName = currentSubDomainQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); + + domain.setAttribute(QUALIFIED_NAME, updatedQualifiedName); + + moveChildrenToAnotherDomain(domainVertex, null, sourceDomainQualifiedName, targetDomainQualifiedName); + + LOG.info("Moved subDomain {} to Domain {}", domainName, targetDomainQualifiedName); + + } finally { + RequestContext.get().endMetricRecord(recorder); + } + } + + private void moveChildrenToAnotherDomain(AtlasVertex childDomainVertex, + String parentDomainQualifiedName, + String sourceDomainQualifiedName, + String targetDomainQualifiedName) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("moveChildrenToAnotherDomain"); + + + try { + LOG.info("Moving child domain {} to Domain {}", childDomainVertex.getProperty(NAME, String.class), targetDomainQualifiedName); + Map updatedAttributes = new HashMap<>(); + + String currentDomainQualifiedName = childDomainVertex.getProperty(QUALIFIED_NAME, String.class); + String updatedQualifiedName = currentDomainQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); + + // Change domain qualifiedName + childDomainVertex.setProperty(QUALIFIED_NAME, updatedQualifiedName); + updatedAttributes.put(QUALIFIED_NAME, updatedQualifiedName); + + //change superDomainQN, parentDomainQN + childDomainVertex.setProperty(SUPER_DOMAIN_QN, targetDomainQualifiedName); + childDomainVertex.setProperty(PARENT_DOMAIN_QN, parentDomainQualifiedName); + + //update system properties + GraphHelper.setModifiedByAsString(childDomainVertex, RequestContext.get().getUser()); + GraphHelper.setModifiedTime(childDomainVertex, System.currentTimeMillis()); + + // move products to target Domain + Iterator products = getActiveChildrenVertices(childDomainVertex, DATA_PRODUCT_EDGE_LABEL); + + while (products.hasNext()) { + AtlasVertex productVertex = products.next(); + moveChildDataProductToAnotherDomain(productVertex, sourceDomainQualifiedName, targetDomainQualifiedName); + } + + // Get all children domains of current domain + Iterator childDomains = getActiveChildrenVertices(childDomainVertex, DOMAIN_PARENT_EDGE_LABEL); + + while (childDomains.hasNext()) { + AtlasVertex childVertex = childDomains.next(); + moveChildrenToAnotherDomain(childVertex, updatedQualifiedName, sourceDomainQualifiedName, targetDomainQualifiedName); + } + + recordUpdatedChildEntities(childDomainVertex, updatedAttributes); + + LOG.info("Moved child domain {} to Domain {}", childDomainVertex.getProperty(NAME, String.class), targetDomainQualifiedName); + } finally { + RequestContext.get().endMetricRecord(recorder); + } + } + + private void moveChildDataProductToAnotherDomain(AtlasVertex productVertex, + String sourceDomainQualifiedName, + String targetDomainQualifiedName) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("moveChildDataProductToAnotherDomain"); + + try { + String productName = productVertex.getProperty(NAME, String.class); + LOG.info("Moving dataProduct {} to Domain {}", productName, targetDomainQualifiedName); + Map updatedAttributes = new HashMap<>(); + + String currentDomainQualifiedName = productVertex.getProperty(PARENT_DOMAIN_QN, String.class); + String updatedQualifiedName = currentDomainQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); + + productVertex.setProperty(QUALIFIED_NAME, updatedQualifiedName); + updatedAttributes.put(QUALIFIED_NAME, updatedQualifiedName); + + productVertex.setProperty(PARENT_DOMAIN_QN, targetDomainQualifiedName); + + //update system properties + GraphHelper.setModifiedByAsString(productVertex, RequestContext.get().getUser()); + GraphHelper.setModifiedTime(productVertex, System.currentTimeMillis()); + + recordUpdatedChildEntities(productVertex, updatedAttributes); + + LOG.info("Moved dataProduct {} to Domain {}", productName, targetDomainQualifiedName); + } finally { + RequestContext.get().endMetricRecord(recorder); + } + } + + private void setParent(AtlasEntity entity, EntityMutationContext context) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("DomainPreProcessor.setParent"); + if (parentDomain == null) { + AtlasObjectId objectId = (AtlasObjectId) entity.getRelationshipAttribute(PARENT_DOMAIN); + + if (objectId != null) { + if (StringUtils.isNotEmpty(objectId.getGuid())) { + AtlasVertex vertex = context.getVertex(objectId.getGuid()); + + if (vertex == null) { + parentDomain = entityRetriever.toAtlasEntityHeader(objectId.getGuid()); + } else { + parentDomain = entityRetriever.toAtlasEntityHeader(vertex); + } + + } else if (MapUtils.isNotEmpty(objectId.getUniqueAttributes()) && + StringUtils.isNotEmpty((String) objectId.getUniqueAttributes().get(QUALIFIED_NAME))) { + parentDomain = new AtlasEntityHeader(objectId.getTypeName(), objectId.getUniqueAttributes()); + + } + } + } + RequestContext.get().endMetricRecord(metricRecorder); + } + + private void domainExists(String domainName, String domainQualifiedName) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("domainExists"); + + boolean exists = false; + try { + List mustClauseList = new ArrayList(); + mustClauseList.add(mapOf("term", mapOf("__dataDomain", domainQualifiedName))); + mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", DATA_DOMAIN_ENTITY_TYPE))); + mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); + mustClauseList.add(mapOf("term", mapOf("name.keyword", domainName))); + + + Map bool = new HashMap<>(); + if (parentDomain != null) { + String parentQname = (String) parentDomain.getAttribute(QUALIFIED_NAME); + mustClauseList.add(mapOf("term", mapOf("__parentDomain", parentQname))); + } else { + List mustNotClauseList = new ArrayList(); + mustNotClauseList.add(mapOf("exists", mapOf("field", "__parentDomain"))); + bool.put("must_not", mustNotClauseList); + } + + bool.put("must", mustClauseList); + + Map dsl = mapOf("query", mapOf("bool", bool)); + + List domains = indexSearchPaginated(dsl); + + if (CollectionUtils.isNotEmpty(domains)) { + for (AtlasEntityHeader domain : domains) { + String name = (String) domain.getAttribute(NAME); + if (domainName.equals(name)) { + exists = true; + break; + } + } + } + } finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + + if (exists) { + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, domainName); + } + } +} + + From 9bcd34ea837a375cdc6265ed9a3417549cd4eff4 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Tue, 12 Mar 2024 00:00:39 +0530 Subject: [PATCH 002/316] uthorized function updated --- pom.xml | 2 +- .../v2/preprocessor/datamesh/AbstractDomainPreProcessor.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pom.xml b/pom.xml index 1cc9aa70dc9..3abc39041b7 100644 --- a/pom.xml +++ b/pom.xml @@ -717,7 +717,7 @@ 4.4.13 2.12.4 2.12.4 - 0.6.03 + 0.6.04 0.5.3 1 3.1.0 diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 5f903caeb5e..546d6e63bcd 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -116,10 +116,10 @@ protected void isAuthorized(AtlasEntityHeader sourceDomain, AtlasEntityHeader ta "create on target Domain: ", targetDomain.getAttribute(NAME)); AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, targetDomain), - "update on target Glossary: ", targetDomain.getAttribute(NAME)); + "update on target Domain: ", targetDomain.getAttribute(NAME)); AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_DELETE, targetDomain), - "delete on target Glossary: ", targetDomain.getAttribute(NAME)); + "delete on target Domain: ", targetDomain.getAttribute(NAME)); } /** From c7a988431926a8af9a1a7afef41a3d5844098998 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Tue, 12 Mar 2024 00:08:39 +0530 Subject: [PATCH 003/316] janusgraph version updated --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3abc39041b7..1cc9aa70dc9 100644 --- a/pom.xml +++ b/pom.xml @@ -717,7 +717,7 @@ 4.4.13 2.12.4 2.12.4 - 0.6.04 + 0.6.03 0.5.3 1 3.1.0 From 84aa0885fd3fcf97c128f218066c014ef4fc82a4 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Tue, 12 Mar 2024 13:26:03 +0530 Subject: [PATCH 004/316] support for only UPDATE operation --- .../preprocessor/datamesh/DataProductPreProcessor.java | 9 +++++++-- .../v2/preprocessor/datamesh/DomainPreProcessor.java | 9 +++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index b3f596439cb..ccdda0ba569 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -39,7 +39,7 @@ public DataProductPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetrie public void processAttributes(AtlasStruct entityStruct, EntityMutationContext context, EntityMutations.EntityOperation operation) throws AtlasBaseException { //Handle name & qualifiedName - if (LOG.isDebugEnabled()) { + if (operation == EntityMutations.EntityOperation.UPDATE && LOG.isDebugEnabled()) { LOG.debug("DataProductPreProcessor.processAttributes: pre processing {}, {}", entityStruct.getAttribute(QUALIFIED_NAME), operation); } @@ -50,7 +50,12 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co AtlasVertex vertex = context.getVertex(entity.getGuid()); setParent(entity, context); - processUpdateDomain(entity, vertex); + + if (operation == EntityMutations.EntityOperation.UPDATE) { + processUpdateDomain(entity, vertex); + } else { + LOG.error("DataProductPreProcessor.processAttributes: Operation not supported {}", operation); + } } private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index fd64c719f5d..eb6ff2eb3d1 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -68,7 +68,7 @@ public DomainPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever e public void processAttributes(AtlasStruct entityStruct, EntityMutationContext context, EntityMutations.EntityOperation operation) throws AtlasBaseException { //Handle name & qualifiedName - if (LOG.isDebugEnabled()) { + if (operation == EntityMutations.EntityOperation.UPDATE && LOG.isDebugEnabled()) { LOG.debug("DomainPreProcessor.processAttributes: pre processing {}, {}", entityStruct.getAttribute(QUALIFIED_NAME), operation); } @@ -79,7 +79,12 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co AtlasVertex vertex = context.getVertex(entity.getGuid()); setParent(entity, context); - processUpdateDomain(entity, vertex); + + if (operation == EntityMutations.EntityOperation.UPDATE) { + processUpdateDomain(entity, vertex); + } else { + LOG.error("DataProductPreProcessor.processAttributes: Operation not supported {}", operation); + } } private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { From 9e6963df482a6df046fbd67601f0b9ef231e201e Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Wed, 13 Mar 2024 11:58:43 +0530 Subject: [PATCH 005/316] update superdomain with parentdomain --- .../datamesh/DataProductPreProcessor.java | 41 +++++++++---------- .../datamesh/DomainPreProcessor.java | 23 ++++++----- 2 files changed, 31 insertions(+), 33 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index ccdda0ba569..92727dd6691 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -23,8 +23,7 @@ import java.util.Map; import static org.apache.atlas.repository.Constants.*; -import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.PARENT_DOMAIN; -import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.PARENT_DOMAIN_QN; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.*; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; public class DataProductPreProcessor extends AbstractDomainPreProcessor { @@ -63,9 +62,6 @@ private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws String productName = (String) entity.getAttribute(NAME); String vertexQnName = vertex.getProperty(QUALIFIED_NAME, String.class); -// if (StringUtils.isEmpty(catName) || isNameInvalid(catName)) { -// throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); -// } AtlasEntity storedProduct = entityRetriever.toAtlasEntity(vertex); AtlasRelatedObjectId currentDomain = (AtlasRelatedObjectId) storedProduct.getRelationshipAttribute(PARENT_DOMAIN); @@ -73,20 +69,20 @@ private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws String currentDomainQualifiedName = (String) currentDomainHeader.getAttribute(QUALIFIED_NAME); String newDomainQualifiedName = (String) parentDomain.getAttribute(QUALIFIED_NAME); + String superDomainQualifiedName = (String) parentDomain.getAttribute(SUPER_DOMAIN_QN); if (!currentDomainQualifiedName.equals(newDomainQualifiedName)) { //Auth check isAuthorized(currentDomainHeader, parentDomain); - processMoveDataProductToAnotherDomain(entity, currentDomainQualifiedName, newDomainQualifiedName, vertexQnName); + processMoveDataProductToAnotherDomain(entity, currentDomainQualifiedName, newDomainQualifiedName, vertexQnName, superDomainQualifiedName); entity.setAttribute(PARENT_DOMAIN_QN, newDomainQualifiedName); } else { String vertexName = vertex.getProperty(NAME, String.class); if (!vertexName.equals(productName)) { - domainExists(productName, newDomainQualifiedName); + productExists(productName, newDomainQualifiedName); } - entity.setAttribute(QUALIFIED_NAME, vertexQnName); } @@ -96,7 +92,8 @@ private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws private void processMoveDataProductToAnotherDomain(AtlasEntity product, String sourceDomainQualifiedName, String targetDomainQualifiedName, - String currentDataProductQualifiedName) throws AtlasBaseException { + String currentDataProductQualifiedName, + String superDomainQualifiedName) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("processMoveDataProductToAnotherDomain"); try { @@ -104,11 +101,13 @@ private void processMoveDataProductToAnotherDomain(AtlasEntity product, LOG.info("Moving dataProduct {} to Domain {}", domainName, targetDomainQualifiedName); - domainExists(domainName, targetDomainQualifiedName); + productExists(domainName, targetDomainQualifiedName); String updatedQualifiedName = currentDataProductQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); product.setAttribute(QUALIFIED_NAME, updatedQualifiedName); + product.setAttribute(PARENT_DOMAIN_QN, targetDomainQualifiedName); + product.setAttribute(SUPER_DOMAIN_QN, superDomainQualifiedName); LOG.info("Moved dataProduct {} to Domain {}", domainName, targetDomainQualifiedName); @@ -142,25 +141,23 @@ private void setParent(AtlasEntity entity, EntityMutationContext context) throws RequestContext.get().endMetricRecord(metricRecorder); } - private void domainExists(String domainName, String domainQualifiedName) throws AtlasBaseException { + private void productExists(String productName, String parentDomainQualifiedName) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("domainExists"); boolean exists = false; try { List mustClauseList = new ArrayList(); - mustClauseList.add(mapOf("term", mapOf("__dataDomain", domainQualifiedName))); mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", DATA_DOMAIN_ENTITY_TYPE))); mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); - mustClauseList.add(mapOf("term", mapOf("name.keyword", domainName))); + mustClauseList.add(mapOf("term", mapOf("name.keyword", productName))); Map bool = new HashMap<>(); if (parentDomain != null) { - String parentQname = (String) parentDomain.getAttribute(QUALIFIED_NAME); - mustClauseList.add(mapOf("term", mapOf("__parentDomain", parentQname))); + mustClauseList.add(mapOf("term", mapOf("parentDomainQualifiedName", parentDomainQualifiedName))); } else { List mustNotClauseList = new ArrayList(); - mustNotClauseList.add(mapOf("exists", mapOf("field", "__parentDomain"))); + mustNotClauseList.add(mapOf("exists", mapOf("field", "parentDomainQualifiedName"))); bool.put("must_not", mustNotClauseList); } @@ -168,12 +165,12 @@ private void domainExists(String domainName, String domainQualifiedName) throws Map dsl = mapOf("query", mapOf("bool", bool)); - List domains = indexSearchPaginated(dsl); + List products = indexSearchPaginated(dsl); - if (CollectionUtils.isNotEmpty(domains)) { - for (AtlasEntityHeader domain : domains) { - String name = (String) domain.getAttribute(NAME); - if (domainName.equals(name)) { + if (CollectionUtils.isNotEmpty(products)) { + for (AtlasEntityHeader product : products) { + String name = (String) product.getAttribute(NAME); + if (productName.equals(name)) { exists = true; break; } @@ -184,7 +181,7 @@ private void domainExists(String domainName, String domainQualifiedName) throws } if (exists) { - throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, domainName); + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, productName); } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index eb6ff2eb3d1..3c6fce63baa 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -101,14 +101,14 @@ private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws String currentDomainQualifiedName = (String) currentDomainHeader.getAttribute(QUALIFIED_NAME); String newDomainQualifiedName = (String) parentDomain.getAttribute(QUALIFIED_NAME); + String superDomainQualifiedName = (String) parentDomain.getAttribute(SUPER_DOMAIN_QN); if (!currentDomainQualifiedName.equals(newDomainQualifiedName)) { //Auth check isAuthorized(currentDomainHeader, parentDomain); - processMoveSubDomainToAnotherDomain(entity, vertex, currentDomainQualifiedName, newDomainQualifiedName, vertexQnName); - entity.setAttribute(PARENT_DOMAIN_QN, newDomainQualifiedName); + processMoveSubDomainToAnotherDomain(entity, vertex, currentDomainQualifiedName, newDomainQualifiedName, vertexQnName, superDomainQualifiedName); } else { String vertexName = vertex.getProperty(NAME, String.class); @@ -126,7 +126,8 @@ private void processMoveSubDomainToAnotherDomain(AtlasEntity domain, AtlasVertex domainVertex, String sourceDomainQualifiedName, String targetDomainQualifiedName, - String currentSubDomainQualifiedName) throws AtlasBaseException { + String currentSubDomainQualifiedName, + String superDomainQualifiedName) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("processMoveSubDomainToAnotherGlossary"); try { @@ -139,8 +140,10 @@ private void processMoveSubDomainToAnotherDomain(AtlasEntity domain, String updatedQualifiedName = currentSubDomainQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); domain.setAttribute(QUALIFIED_NAME, updatedQualifiedName); + domain.setAttribute(PARENT_DOMAIN_QN, targetDomainQualifiedName); + domain.setAttribute(SUPER_DOMAIN_QN, superDomainQualifiedName); - moveChildrenToAnotherDomain(domainVertex, null, sourceDomainQualifiedName, targetDomainQualifiedName); + moveChildrenToAnotherDomain(domainVertex, superDomainQualifiedName, sourceDomainQualifiedName, targetDomainQualifiedName); LOG.info("Moved subDomain {} to Domain {}", domainName, targetDomainQualifiedName); @@ -168,8 +171,8 @@ private void moveChildrenToAnotherDomain(AtlasVertex childDomainVertex, updatedAttributes.put(QUALIFIED_NAME, updatedQualifiedName); //change superDomainQN, parentDomainQN - childDomainVertex.setProperty(SUPER_DOMAIN_QN, targetDomainQualifiedName); - childDomainVertex.setProperty(PARENT_DOMAIN_QN, parentDomainQualifiedName); + childDomainVertex.setProperty(SUPER_DOMAIN_QN, parentDomainQualifiedName); + childDomainVertex.setProperty(PARENT_DOMAIN_QN, targetDomainQualifiedName); //update system properties GraphHelper.setModifiedByAsString(childDomainVertex, RequestContext.get().getUser()); @@ -254,13 +257,12 @@ private void setParent(AtlasEntity entity, EntityMutationContext context) throws RequestContext.get().endMetricRecord(metricRecorder); } - private void domainExists(String domainName, String domainQualifiedName) throws AtlasBaseException { + private void domainExists(String domainName, String parentDomainQualifiedName) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("domainExists"); boolean exists = false; try { List mustClauseList = new ArrayList(); - mustClauseList.add(mapOf("term", mapOf("__dataDomain", domainQualifiedName))); mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", DATA_DOMAIN_ENTITY_TYPE))); mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); mustClauseList.add(mapOf("term", mapOf("name.keyword", domainName))); @@ -268,11 +270,10 @@ private void domainExists(String domainName, String domainQualifiedName) throws Map bool = new HashMap<>(); if (parentDomain != null) { - String parentQname = (String) parentDomain.getAttribute(QUALIFIED_NAME); - mustClauseList.add(mapOf("term", mapOf("__parentDomain", parentQname))); + mustClauseList.add(mapOf("term", mapOf("parentDomainQualifiedName", parentDomainQualifiedName))); } else { List mustNotClauseList = new ArrayList(); - mustNotClauseList.add(mapOf("exists", mapOf("field", "__parentDomain"))); + mustNotClauseList.add(mapOf("exists", mapOf("field", "parentDomainQualifiedName"))); bool.put("must_not", mustNotClauseList); } From e893e1ef65e3a29a80fad3d713ef12d9a1ceec8c Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Wed, 13 Mar 2024 18:37:04 +0530 Subject: [PATCH 006/316] edge label changed --- .../java/org/apache/atlas/repository/Constants.java | 4 ++-- .../v2/preprocessor/datamesh/DomainPreProcessor.java | 10 ++++++---- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 37b6d7416b8..42ad0d9e26e 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -139,8 +139,8 @@ public final class Constants { */ public static final String DATA_DOMAIN_ENTITY_TYPE = "DataDomain"; public static final String DATA_PRODUCT_ENTITY_TYPE = "DataProduct"; - public static final String DATA_PRODUCT_EDGE_LABEL = "r:AtlasDataProduct"; - public static final String DOMAIN_PARENT_EDGE_LABEL = "r:AtlasDataDomainHierarchyLink"; + public static final String DATA_PRODUCT_EDGE_LABEL = "__DataDomain.dataProducts"; + public static final String DOMAIN_PARENT_EDGE_LABEL = "__DataDomain.subDomains"; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index 3c6fce63baa..def84b56ff1 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -107,7 +107,7 @@ private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws if (!currentDomainQualifiedName.equals(newDomainQualifiedName)) { //Auth check isAuthorized(currentDomainHeader, parentDomain); - + LOG.info("Edge Labels: {}", entity.getRelationshipAttributes()); processMoveSubDomainToAnotherDomain(entity, vertex, currentDomainQualifiedName, newDomainQualifiedName, vertexQnName, superDomainQualifiedName); } else { @@ -171,8 +171,8 @@ private void moveChildrenToAnotherDomain(AtlasVertex childDomainVertex, updatedAttributes.put(QUALIFIED_NAME, updatedQualifiedName); //change superDomainQN, parentDomainQN - childDomainVertex.setProperty(SUPER_DOMAIN_QN, parentDomainQualifiedName); - childDomainVertex.setProperty(PARENT_DOMAIN_QN, targetDomainQualifiedName); + childDomainVertex.setProperty(SUPER_DOMAIN_QN, targetDomainQualifiedName); + childDomainVertex.setProperty(PARENT_DOMAIN_QN, parentDomainQualifiedName); //update system properties GraphHelper.setModifiedByAsString(childDomainVertex, RequestContext.get().getUser()); @@ -183,7 +183,7 @@ private void moveChildrenToAnotherDomain(AtlasVertex childDomainVertex, while (products.hasNext()) { AtlasVertex productVertex = products.next(); - moveChildDataProductToAnotherDomain(productVertex, sourceDomainQualifiedName, targetDomainQualifiedName); + moveChildDataProductToAnotherDomain(productVertex, parentDomainQualifiedName, sourceDomainQualifiedName, targetDomainQualifiedName); } // Get all children domains of current domain @@ -203,6 +203,7 @@ private void moveChildrenToAnotherDomain(AtlasVertex childDomainVertex, } private void moveChildDataProductToAnotherDomain(AtlasVertex productVertex, + String parentDomainQualifiedName, String sourceDomainQualifiedName, String targetDomainQualifiedName) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("moveChildDataProductToAnotherDomain"); @@ -219,6 +220,7 @@ private void moveChildDataProductToAnotherDomain(AtlasVertex productVertex, updatedAttributes.put(QUALIFIED_NAME, updatedQualifiedName); productVertex.setProperty(PARENT_DOMAIN_QN, targetDomainQualifiedName); + productVertex.setProperty(SUPER_DOMAIN_QN, parentDomainQualifiedName); //update system properties GraphHelper.setModifiedByAsString(productVertex, RequestContext.get().getUser()); From 81fcc6a27ae37ec4aa2f47ac01f5ff5ae2c510c6 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Wed, 13 Mar 2024 18:41:46 +0530 Subject: [PATCH 007/316] logs removed --- .../graph/v2/preprocessor/datamesh/DomainPreProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index def84b56ff1..f583508ffbb 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -107,7 +107,7 @@ private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws if (!currentDomainQualifiedName.equals(newDomainQualifiedName)) { //Auth check isAuthorized(currentDomainHeader, parentDomain); - LOG.info("Edge Labels: {}", entity.getRelationshipAttributes()); + processMoveSubDomainToAnotherDomain(entity, vertex, currentDomainQualifiedName, newDomainQualifiedName, vertexQnName, superDomainQualifiedName); } else { From 4c4d4e3e55cfdf33b639bc9b2d509a534d1e4c06 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Wed, 13 Mar 2024 18:43:24 +0530 Subject: [PATCH 008/316] comments removed --- .../graph/v2/preprocessor/datamesh/DataProductPreProcessor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 92727dd6691..c6cf4bf3287 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -43,7 +43,6 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co entityStruct.getAttribute(QUALIFIED_NAME), operation); } -// this.context = context; AtlasEntity entity = (AtlasEntity) entityStruct; AtlasVertex vertex = context.getVertex(entity.getGuid()); From 6baaabe8b63756b69963df667c9c6a1856803819 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Thu, 14 Mar 2024 10:51:09 +0530 Subject: [PATCH 009/316] fix: make tag properties consistent --- .../store/graph/AtlasEntityStore.java | 5 +++ .../store/graph/v2/AtlasEntityStoreV2.java | 28 ++++++++++++ .../graph/v2/ClassificationAssociator.java | 13 ++++++ .../store/graph/v2/EntityGraphMapper.java | 43 +++++++++++++++++++ .../org/apache/atlas/web/rest/EntityREST.java | 20 +++++++++ 5 files changed, 109 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java index 02022fe0401..960ec3808e4 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java @@ -266,6 +266,11 @@ EntityMutationResponse deleteByUniqueAttributes(List objectIds) */ EntityMutationResponse deleteByIds(List guid) throws AtlasBaseException; + /* + * Repair classification mappings + */ + public void repairClassificationMappings(final String guid) throws AtlasBaseException; + /* * Return list of deleted entity guids */ diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 6e3487f8d37..ebcc59674bd 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -913,6 +913,34 @@ public String getGuidByUniqueAttributes(AtlasEntityType entityType, Map repairClassificationMappings({})", guid); + } + + if (StringUtils.isEmpty(guid)) { + throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guid); + } + + AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(graph, guid); + + if (entityVertex == null) { + throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guid); + } + + AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeaderWithClassifications(entityVertex); + + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE_CLASSIFICATION, entityHeader), "repair classification mappings: guid=", guid); + + entityGraphMapper.repairClassificationMappings(entityHeader, entityVertex); + + if (LOG.isDebugEnabled()) { + LOG.debug("<== repairClassificationMappings({})", guid); + } + } + @Override @GraphTransaction public void addClassifications(final String guid, final List classifications) throws AtlasBaseException { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java index 85bb85fe9bc..d21b860740d 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java @@ -154,6 +154,19 @@ public Updater(AtlasTypeRegistry typeRegistry, AtlasEntityStore entitiesStore, this(AtlasGraphProvider.getGraphInstance(), typeRegistry, entitiesStore, entityGraphMapper, entityChangeNotifier, instanceConverter); } + public void repairClassificationsMappings(List guids) throws AtlasBaseException { + + for (String guid : guids) { + // Fetch attached classifications + AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(this.graph, guid); + List classifications = entityRetriever.getAllClassifications(vertex); + for (AtlasClassification classification : classifications) { + // Fetch the entity header + + } + } + } + public void setClassifications(Map map) throws AtlasBaseException { RequestContext.get().setDelayTagNotifications(true); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index f624e30dc98..7e0b8866f50 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -2979,6 +2979,49 @@ private void updateInConsistentOwnedMapVertices(AttributeMutationContext ctx, At } } + public void repairClassificationMappings(AtlasEntityHeader entityHeader, AtlasVertex entityVertex) throws AtlasBaseException { + if (entityHeader.getClassifications() != null) { + List classifications = entityHeader.getClassifications(); + String propagatedClassificationName = null; + String classificationNames = null; + String classificationText = null; + List propagatedClassificationNamesList = new ArrayList<>(); + List classificationNamesList = new ArrayList<>(); + for (AtlasClassification classification : classifications) { + String typeName = classification.getTypeName(); + classificationText = fullTextMapperV2.getClassificationTextForEntity(new AtlasEntity(entityHeader)); + if(classification.isPropagate()) { + //Add to propagatedClassificationNamesList + propagatedClassificationNamesList.add(typeName); + //Create delimiter separated string for propagatedClassificationNames + if (StringUtils.isEmpty(propagatedClassificationName)) { + propagatedClassificationName = CLASSIFICATION_NAME_DELIMITER + typeName + CLASSIFICATION_NAME_DELIMITER; + } else { + propagatedClassificationName = propagatedClassificationName + typeName + CLASSIFICATION_NAME_DELIMITER; + } + } else { + //Add to classificationNamesList + classificationNamesList.add(typeName); + //Create delimiter separated string for classificationNames + if (StringUtils.isEmpty(classificationNames)) { + classificationNames = CLASSIFICATION_NAME_DELIMITER + typeName + CLASSIFICATION_NAME_DELIMITER; + } else { + classificationNames = classificationNames + typeName + CLASSIFICATION_NAME_DELIMITER; + } + } + } + + //Update classificationNames and propagatedClassificationNames in entityVertex + AtlasGraphUtilsV2.setEncodedProperty(entityVertex, CLASSIFICATION_NAMES_KEY, classificationNames); + AtlasGraphUtilsV2.setEncodedProperty(entityVertex, CLASSIFICATION_TEXT_KEY, classificationText); + AtlasGraphUtilsV2.setEncodedProperty(entityVertex, PROPAGATED_CLASSIFICATION_NAMES_KEY, propagatedClassificationName); + + //Update classificationNames and propagatedClassificationNames in entityHeader + AtlasGraphUtilsV2.setEncodedProperty(entityVertex, TRAIT_NAMES_PROPERTY_KEY, classificationNamesList); + AtlasGraphUtilsV2.setEncodedProperty(entityVertex, PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, propagatedClassificationNamesList); + } + } + public void addClassifications(final EntityMutationContext context, String guid, List classifications) throws AtlasBaseException { if (CollectionUtils.isNotEmpty(classifications)) { MetricRecorder metric = RequestContext.get().startMetricRecord("addClassifications"); diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java index ee6846fd8ff..6b3b1b52b37 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java @@ -1274,6 +1274,26 @@ public void setClassifications(AtlasEntityHeaders entityHeaders) throws AtlasBas } } + @POST + @Path("bulk/repairClassificationsMappings") + @Produces(Servlets.JSON_MEDIA_TYPE) + @Consumes(Servlets.JSON_MEDIA_TYPE) + @Timed + public void repairClassifications(List guids) throws AtlasBaseException { + AtlasPerfTracer perf = null; + + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityREST.repairClassifications()"); + } + + ClassificationAssociator.Updater associator = new ClassificationAssociator.Updater(typeRegistry, entitiesStore, entityGraphMapper, entityChangeNotifier, instanceConverter); + associator.repairClassificationsMappings(guids); + } finally { + AtlasPerfTracer.log(perf); + } + } + @POST @Path("/guid/{guid}/businessmetadata") @Produces(Servlets.JSON_MEDIA_TYPE) From b101373db6e2fe7643588a87aaa2e8714e4e2c6f Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Mon, 18 Mar 2024 13:04:03 +0530 Subject: [PATCH 010/316] Root domain move case --- .../graph/v2/preprocessor/datamesh/DomainPreProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index f583508ffbb..996aa4efb49 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -93,7 +93,7 @@ private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws String vertexQnName = vertex.getProperty(QUALIFIED_NAME, String.class); AtlasEntity storedDomain = entityRetriever.toAtlasEntity(vertex); - if(storedDomain.getRelationshipAttribute(PARENT_DOMAIN) == null){ + if(vertex.getProperty(PARENT_DOMAIN, String.class) == null){ throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Cannot move Root Domain"); } AtlasRelatedObjectId currentDomain = (AtlasRelatedObjectId) storedDomain.getRelationshipAttribute(PARENT_DOMAIN); From a332c8aba20d7fcf52f4574d7b763443099859f1 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Mon, 18 Mar 2024 13:32:43 +0530 Subject: [PATCH 011/316] Root domain move issue resolved --- .../graph/v2/preprocessor/datamesh/DomainPreProcessor.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index 996aa4efb49..61b73c4c56d 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -93,9 +93,6 @@ private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws String vertexQnName = vertex.getProperty(QUALIFIED_NAME, String.class); AtlasEntity storedDomain = entityRetriever.toAtlasEntity(vertex); - if(vertex.getProperty(PARENT_DOMAIN, String.class) == null){ - throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Cannot move Root Domain"); - } AtlasRelatedObjectId currentDomain = (AtlasRelatedObjectId) storedDomain.getRelationshipAttribute(PARENT_DOMAIN); AtlasEntityHeader currentDomainHeader = entityRetriever.toAtlasEntityHeader(currentDomain.getGuid()); String currentDomainQualifiedName = (String) currentDomainHeader.getAttribute(QUALIFIED_NAME); @@ -105,6 +102,10 @@ private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws if (!currentDomainQualifiedName.equals(newDomainQualifiedName)) { + if(storedDomain.getRelationshipAttribute(PARENT_DOMAIN) == null){ + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Cannot move Root Domain"); + } + //Auth check isAuthorized(currentDomainHeader, parentDomain); From 96f5bb23e0cdc4091319394cbca06112415c68a8 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Tue, 19 Mar 2024 17:59:23 +0530 Subject: [PATCH 012/316] Null Exceptions handled --- .../v2/preprocessor/PreProcessorUtils.java | 1 + .../datamesh/DataProductPreProcessor.java | 78 ++++++++++++++----- .../datamesh/DomainPreProcessor.java | 32 +++++--- 3 files changed, 82 insertions(+), 29 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index b23ab7c3f9b..57fd775d4af 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -37,6 +37,7 @@ public class PreProcessorUtils { public static final String PARENT_DOMAIN = "parentDomain"; public static final String PARENT_DOMAIN_QN = "parentDomainQualifiedName"; public static final String SUPER_DOMAIN_QN = "superDomainQualifiedName"; + public static final String DATA_DOMAIN = "dataDomain"; //Query models constants public static final String PREFIX_QUERY_QN = "default/collection/"; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index c6cf4bf3287..0d458816292 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -29,6 +29,7 @@ public class DataProductPreProcessor extends AbstractDomainPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(DomainPreProcessor.class); private AtlasEntityHeader parentDomain; + private EntityMutationContext context; public DataProductPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, AtlasGraph graph, EntityGraphMapper entityGraphMapper) { super(typeRegistry, entityRetriever, graph); @@ -42,7 +43,7 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co LOG.debug("DataProductPreProcessor.processAttributes: pre processing {}, {}", entityStruct.getAttribute(QUALIFIED_NAME), operation); } - + this.context = context; AtlasEntity entity = (AtlasEntity) entityStruct; AtlasVertex vertex = context.getVertex(entity.getGuid()); @@ -61,26 +62,39 @@ private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws String productName = (String) entity.getAttribute(NAME); String vertexQnName = vertex.getProperty(QUALIFIED_NAME, String.class); + AtlasEntityHeader currentParentDomainHeader = null; + String currentParentDomainQualifiedName = ""; AtlasEntity storedProduct = entityRetriever.toAtlasEntity(vertex); - AtlasRelatedObjectId currentDomain = (AtlasRelatedObjectId) storedProduct.getRelationshipAttribute(PARENT_DOMAIN); - AtlasEntityHeader currentDomainHeader = entityRetriever.toAtlasEntityHeader(currentDomain.getGuid()); - String currentDomainQualifiedName = (String) currentDomainHeader.getAttribute(QUALIFIED_NAME); + AtlasRelatedObjectId currentParentDomain = (AtlasRelatedObjectId) storedProduct.getRelationshipAttribute(DATA_DOMAIN); + + String newParentDomainQualifiedName = ""; + String superDomainQualifiedName = ""; + + if(currentParentDomain != null){ + currentParentDomainHeader = entityRetriever.toAtlasEntityHeader(currentParentDomain.getGuid()); + currentParentDomainQualifiedName = (String) currentParentDomainHeader.getAttribute(QUALIFIED_NAME); + } - String newDomainQualifiedName = (String) parentDomain.getAttribute(QUALIFIED_NAME); - String superDomainQualifiedName = (String) parentDomain.getAttribute(SUPER_DOMAIN_QN); + if (parentDomain != null) { + newParentDomainQualifiedName = (String) parentDomain.getAttribute(QUALIFIED_NAME); + superDomainQualifiedName = (String) parentDomain.getAttribute(SUPER_DOMAIN_QN); + if(superDomainQualifiedName == null){ + superDomainQualifiedName = newParentDomainQualifiedName; + } + } - if (!currentDomainQualifiedName.equals(newDomainQualifiedName)) { + if (!currentParentDomainQualifiedName.equals(newParentDomainQualifiedName) && !newParentDomainQualifiedName.isEmpty()) { //Auth check - isAuthorized(currentDomainHeader, parentDomain); + isAuthorized(currentParentDomainHeader, parentDomain); - processMoveDataProductToAnotherDomain(entity, currentDomainQualifiedName, newDomainQualifiedName, vertexQnName, superDomainQualifiedName); - entity.setAttribute(PARENT_DOMAIN_QN, newDomainQualifiedName); + processMoveDataProductToAnotherDomain(entity, currentParentDomainQualifiedName, newParentDomainQualifiedName, vertexQnName, superDomainQualifiedName); + entity.setAttribute(PARENT_DOMAIN_QN, newParentDomainQualifiedName); } else { String vertexName = vertex.getProperty(NAME, String.class); if (!vertexName.equals(productName)) { - productExists(productName, newDomainQualifiedName); + productExists(productName, newParentDomainQualifiedName); } entity.setAttribute(QUALIFIED_NAME, vertexQnName); } @@ -118,24 +132,48 @@ private void processMoveDataProductToAnotherDomain(AtlasEntity product, private void setParent(AtlasEntity entity, EntityMutationContext context) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("DataProductPreProcessor.setParent"); if (parentDomain == null) { - AtlasObjectId objectId = (AtlasObjectId) entity.getRelationshipAttribute(PARENT_DOMAIN); + Object relationshipAttribute = entity.getRelationshipAttribute(PARENT_DOMAIN); + + if(relationshipAttribute instanceof AtlasObjectId){ + AtlasObjectId objectId = (AtlasObjectId) relationshipAttribute; + if (objectId != null) { + if (StringUtils.isNotEmpty(objectId.getGuid())) { + AtlasVertex vertex = context.getVertex(objectId.getGuid()); + + if (vertex == null) { + parentDomain = entityRetriever.toAtlasEntityHeader(objectId.getGuid()); + } else { + parentDomain = entityRetriever.toAtlasEntityHeader(vertex); + } - if (objectId != null) { - if (StringUtils.isNotEmpty(objectId.getGuid())) { - AtlasVertex vertex = context.getVertex(objectId.getGuid()); + } else if (MapUtils.isNotEmpty(objectId.getUniqueAttributes()) && + StringUtils.isNotEmpty((String) objectId.getUniqueAttributes().get(QUALIFIED_NAME))) { + parentDomain = new AtlasEntityHeader(objectId.getTypeName(), objectId.getUniqueAttributes()); + + } + } + } + else if(relationshipAttribute instanceof Map){ + Map relationshipMap = (Map) relationshipAttribute; + if (StringUtils.isNotEmpty((String) relationshipMap.get("guid"))) { + AtlasVertex vertex = context.getVertex((String) relationshipMap.get("guid")); if (vertex == null) { - parentDomain = entityRetriever.toAtlasEntityHeader(objectId.getGuid()); + parentDomain = entityRetriever.toAtlasEntityHeader((String) relationshipMap.get("guid")); } else { parentDomain = entityRetriever.toAtlasEntityHeader(vertex); } - } else if (MapUtils.isNotEmpty(objectId.getUniqueAttributes()) && - StringUtils.isNotEmpty((String) objectId.getUniqueAttributes().get(QUALIFIED_NAME))) { - parentDomain = new AtlasEntityHeader(objectId.getTypeName(), objectId.getUniqueAttributes()); + } + else { + parentDomain = new AtlasEntityHeader((String) relationshipMap.get("typeName"), relationshipMap); } } + else{ + LOG.warn("DataProductPreProcessor.setParent: Invalid relationshipAttribute {}", relationshipAttribute); + } + } RequestContext.get().endMetricRecord(metricRecorder); } @@ -146,7 +184,7 @@ private void productExists(String productName, String parentDomainQualifiedName) boolean exists = false; try { List mustClauseList = new ArrayList(); - mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", DATA_DOMAIN_ENTITY_TYPE))); + mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", DATA_PRODUCT_ENTITY_TYPE))); mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); mustClauseList.add(mapOf("term", mapOf("name.keyword", productName))); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index 61b73c4c56d..374df7afe9c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -92,29 +92,43 @@ private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws String domainName = (String) entity.getAttribute(NAME); String vertexQnName = vertex.getProperty(QUALIFIED_NAME, String.class); + AtlasEntityHeader currentParentDomainHeader = null; + String currentParentDomainQualifiedName = ""; + AtlasEntity storedDomain = entityRetriever.toAtlasEntity(vertex); - AtlasRelatedObjectId currentDomain = (AtlasRelatedObjectId) storedDomain.getRelationshipAttribute(PARENT_DOMAIN); - AtlasEntityHeader currentDomainHeader = entityRetriever.toAtlasEntityHeader(currentDomain.getGuid()); - String currentDomainQualifiedName = (String) currentDomainHeader.getAttribute(QUALIFIED_NAME); + AtlasRelatedObjectId currentParentDomain = (AtlasRelatedObjectId) storedDomain.getRelationshipAttribute(PARENT_DOMAIN); + + String newParentDomainQualifiedName = ""; + String superDomainQualifiedName = ""; + + if(currentParentDomain != null){ + currentParentDomainHeader = entityRetriever.toAtlasEntityHeader(currentParentDomain.getGuid()); + currentParentDomainQualifiedName = (String) currentParentDomainHeader.getAttribute(QUALIFIED_NAME); + } - String newDomainQualifiedName = (String) parentDomain.getAttribute(QUALIFIED_NAME); - String superDomainQualifiedName = (String) parentDomain.getAttribute(SUPER_DOMAIN_QN); + if (parentDomain != null) { + newParentDomainQualifiedName = (String) parentDomain.getAttribute(QUALIFIED_NAME); + superDomainQualifiedName = (String) parentDomain.getAttribute(SUPER_DOMAIN_QN); + if(superDomainQualifiedName == null){ + superDomainQualifiedName = newParentDomainQualifiedName; + } + } - if (!currentDomainQualifiedName.equals(newDomainQualifiedName)) { + if (!currentParentDomainQualifiedName.equals(newParentDomainQualifiedName) && !newParentDomainQualifiedName.isEmpty()) { if(storedDomain.getRelationshipAttribute(PARENT_DOMAIN) == null){ throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Cannot move Root Domain"); } //Auth check - isAuthorized(currentDomainHeader, parentDomain); + isAuthorized(currentParentDomainHeader, parentDomain); - processMoveSubDomainToAnotherDomain(entity, vertex, currentDomainQualifiedName, newDomainQualifiedName, vertexQnName, superDomainQualifiedName); + processMoveSubDomainToAnotherDomain(entity, vertex, currentParentDomainQualifiedName, newParentDomainQualifiedName, vertexQnName, superDomainQualifiedName); } else { String vertexName = vertex.getProperty(NAME, String.class); if (!vertexName.equals(domainName)) { - domainExists(domainName, newDomainQualifiedName); + domainExists(domainName, newParentDomainQualifiedName); } entity.setAttribute(QUALIFIED_NAME, vertexQnName); From 8280a026ff64e9a8369be7e6c6e0bcc98f0ff097 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Wed, 27 Mar 2024 22:47:06 +0530 Subject: [PATCH 013/316] dataProduct move issue resolved --- .../v2/preprocessor/datamesh/DataProductPreProcessor.java | 4 ++-- .../graph/v2/preprocessor/datamesh/DomainPreProcessor.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 0d458816292..7dbf03be850 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -84,7 +84,7 @@ private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws } } - if (!currentParentDomainQualifiedName.equals(newParentDomainQualifiedName) && !newParentDomainQualifiedName.isEmpty()) { + if (!currentParentDomainQualifiedName.equals(newParentDomainQualifiedName) && entity.hasRelationshipAttribute(PARENT_DOMAIN)) { //Auth check isAuthorized(currentParentDomainHeader, parentDomain); @@ -132,7 +132,7 @@ private void processMoveDataProductToAnotherDomain(AtlasEntity product, private void setParent(AtlasEntity entity, EntityMutationContext context) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("DataProductPreProcessor.setParent"); if (parentDomain == null) { - Object relationshipAttribute = entity.getRelationshipAttribute(PARENT_DOMAIN); + Object relationshipAttribute = entity.getRelationshipAttribute(DATA_DOMAIN); if(relationshipAttribute instanceof AtlasObjectId){ AtlasObjectId objectId = (AtlasObjectId) relationshipAttribute; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index 374df7afe9c..34eac3b0aa8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -115,7 +115,7 @@ private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws } - if (!currentParentDomainQualifiedName.equals(newParentDomainQualifiedName) && !newParentDomainQualifiedName.isEmpty()) { + if (!currentParentDomainQualifiedName.equals(newParentDomainQualifiedName) && entity.hasRelationshipAttribute(PARENT_DOMAIN)) { if(storedDomain.getRelationshipAttribute(PARENT_DOMAIN) == null){ throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Cannot move Root Domain"); } From 8e48c5c8e7c11e5c2f1a088964c53c7801058b13 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Thu, 28 Mar 2024 04:31:54 +0530 Subject: [PATCH 014/316] split domain implemented --- .../datamesh/DataProductPreProcessor.java | 2 +- .../datamesh/DomainPreProcessor.java | 45 ++++++++++++------- 2 files changed, 31 insertions(+), 16 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 7dbf03be850..f83dff69138 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -84,7 +84,7 @@ private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws } } - if (!currentParentDomainQualifiedName.equals(newParentDomainQualifiedName) && entity.hasRelationshipAttribute(PARENT_DOMAIN)) { + if (!currentParentDomainQualifiedName.equals(newParentDomainQualifiedName) && entity.hasRelationshipAttribute(DATA_DOMAIN)) { //Auth check isAuthorized(currentParentDomainHeader, parentDomain); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index 34eac3b0aa8..2429b72bf15 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -114,14 +114,15 @@ private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws } } - if (!currentParentDomainQualifiedName.equals(newParentDomainQualifiedName) && entity.hasRelationshipAttribute(PARENT_DOMAIN)) { if(storedDomain.getRelationshipAttribute(PARENT_DOMAIN) == null){ throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Cannot move Root Domain"); } //Auth check - isAuthorized(currentParentDomainHeader, parentDomain); + if(parentDomain != null && currentParentDomainHeader != null){ + isAuthorized(currentParentDomainHeader, parentDomain); + } processMoveSubDomainToAnotherDomain(entity, vertex, currentParentDomainQualifiedName, newParentDomainQualifiedName, vertexQnName, superDomainQualifiedName); @@ -147,18 +148,30 @@ private void processMoveSubDomainToAnotherDomain(AtlasEntity domain, try { String domainName = (String) domain.getAttribute(NAME); + String updatedQualifiedName = ""; LOG.info("Moving subdomain {} to Domain {}", domainName, targetDomainQualifiedName); domainExists(domainName, targetDomainQualifiedName); - String updatedQualifiedName = currentSubDomainQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); - domain.setAttribute(QUALIFIED_NAME, updatedQualifiedName); - domain.setAttribute(PARENT_DOMAIN_QN, targetDomainQualifiedName); - domain.setAttribute(SUPER_DOMAIN_QN, superDomainQualifiedName); + // Move Sub-Domain as root Domain + if(targetDomainQualifiedName.isEmpty()){ + targetDomainQualifiedName = "default"; + updatedQualifiedName = currentSubDomainQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); + domain.setAttribute(QUALIFIED_NAME, updatedQualifiedName); + domain.setAttribute(PARENT_DOMAIN_QN, null); + domain.setAttribute(SUPER_DOMAIN_QN, null); + superDomainQualifiedName = updatedQualifiedName ; + } + else{ + updatedQualifiedName = currentSubDomainQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); + domain.setAttribute(QUALIFIED_NAME, updatedQualifiedName); + domain.setAttribute(PARENT_DOMAIN_QN, targetDomainQualifiedName); + domain.setAttribute(SUPER_DOMAIN_QN, superDomainQualifiedName); + } - moveChildrenToAnotherDomain(domainVertex, superDomainQualifiedName, sourceDomainQualifiedName, targetDomainQualifiedName); + moveChildrenToAnotherDomain(domainVertex, superDomainQualifiedName, null, sourceDomainQualifiedName, targetDomainQualifiedName); LOG.info("Moved subDomain {} to Domain {}", domainName, targetDomainQualifiedName); @@ -168,6 +181,7 @@ private void processMoveSubDomainToAnotherDomain(AtlasEntity domain, } private void moveChildrenToAnotherDomain(AtlasVertex childDomainVertex, + String superDomainQualifiedName, String parentDomainQualifiedName, String sourceDomainQualifiedName, String targetDomainQualifiedName) throws AtlasBaseException { @@ -179,14 +193,14 @@ private void moveChildrenToAnotherDomain(AtlasVertex childDomainVertex, Map updatedAttributes = new HashMap<>(); String currentDomainQualifiedName = childDomainVertex.getProperty(QUALIFIED_NAME, String.class); - String updatedQualifiedName = currentDomainQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); + String updatedDomainQualifiedName = currentDomainQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); // Change domain qualifiedName - childDomainVertex.setProperty(QUALIFIED_NAME, updatedQualifiedName); - updatedAttributes.put(QUALIFIED_NAME, updatedQualifiedName); + childDomainVertex.setProperty(QUALIFIED_NAME, updatedDomainQualifiedName); + updatedAttributes.put(QUALIFIED_NAME, updatedDomainQualifiedName); //change superDomainQN, parentDomainQN - childDomainVertex.setProperty(SUPER_DOMAIN_QN, targetDomainQualifiedName); + childDomainVertex.setProperty(SUPER_DOMAIN_QN, superDomainQualifiedName); childDomainVertex.setProperty(PARENT_DOMAIN_QN, parentDomainQualifiedName); //update system properties @@ -198,7 +212,7 @@ private void moveChildrenToAnotherDomain(AtlasVertex childDomainVertex, while (products.hasNext()) { AtlasVertex productVertex = products.next(); - moveChildDataProductToAnotherDomain(productVertex, parentDomainQualifiedName, sourceDomainQualifiedName, targetDomainQualifiedName); + moveChildDataProductToAnotherDomain(productVertex, superDomainQualifiedName, updatedDomainQualifiedName, sourceDomainQualifiedName, targetDomainQualifiedName); } // Get all children domains of current domain @@ -206,7 +220,7 @@ private void moveChildrenToAnotherDomain(AtlasVertex childDomainVertex, while (childDomains.hasNext()) { AtlasVertex childVertex = childDomains.next(); - moveChildrenToAnotherDomain(childVertex, updatedQualifiedName, sourceDomainQualifiedName, targetDomainQualifiedName); + moveChildrenToAnotherDomain(childVertex, superDomainQualifiedName, updatedDomainQualifiedName, sourceDomainQualifiedName, targetDomainQualifiedName); } recordUpdatedChildEntities(childDomainVertex, updatedAttributes); @@ -218,6 +232,7 @@ private void moveChildrenToAnotherDomain(AtlasVertex childDomainVertex, } private void moveChildDataProductToAnotherDomain(AtlasVertex productVertex, + String superDomainQualifiedName, String parentDomainQualifiedName, String sourceDomainQualifiedName, String targetDomainQualifiedName) throws AtlasBaseException { @@ -234,8 +249,8 @@ private void moveChildDataProductToAnotherDomain(AtlasVertex productVertex, productVertex.setProperty(QUALIFIED_NAME, updatedQualifiedName); updatedAttributes.put(QUALIFIED_NAME, updatedQualifiedName); - productVertex.setProperty(PARENT_DOMAIN_QN, targetDomainQualifiedName); - productVertex.setProperty(SUPER_DOMAIN_QN, parentDomainQualifiedName); + productVertex.setProperty(PARENT_DOMAIN_QN, parentDomainQualifiedName); + productVertex.setProperty(SUPER_DOMAIN_QN, superDomainQualifiedName); //update system properties GraphHelper.setModifiedByAsString(productVertex, RequestContext.get().getUser()); From 27285adcace0575077f373ed2161f0bd3d5e10f9 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Thu, 28 Mar 2024 10:02:29 +0530 Subject: [PATCH 015/316] Add DataContract preprocessor and versioning --- .../apache/atlas/repository/Constants.java | 5 + .../store/graph/v2/AtlasEntityStoreV2.java | 5 + .../AbstractContractPreProcessor.java | 129 +++++++++ .../contract/ContractPreProcessor.java | 245 ++++++++++++++++++ .../contract/ContractVersionUtils.java | 127 +++++++++ .../preprocessor/contract/DataContract.java | 129 +++++++++ 6 files changed, 640 insertions(+) create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 4a2cd5905c7..9fdb249fcc1 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -163,6 +163,11 @@ public final class Constants { public static final String ASSET_README_EDGE_LABEL = "__Asset.readme"; public static final String ASSET_LINK_EDGE_LABEL = "__Asset.links"; + /** + * Contract + */ + public static final String CONTRACT_ENTITY_TYPE = "DataContract"; + /** * Lineage relations. */ diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 6e3487f8d37..4daf9128dae 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -58,6 +58,7 @@ import org.apache.atlas.repository.store.graph.v1.RestoreHandlerV1; import org.apache.atlas.repository.store.graph.v2.preprocessor.AuthPolicyPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.ConnectionPreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.contract.ContractPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.resource.LinkPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.accesscontrol.PersonaPreProcessor; @@ -1835,6 +1836,10 @@ public PreProcessor getPreProcessor(String typeName) { case README_ENTITY_TYPE: preProcessor = new ReadmePreProcessor(typeRegistry, entityRetriever); break; + + case CONTRACT_ENTITY_TYPE: + preProcessor = new ContractPreProcessor(graph, typeRegistry, entityRetriever, this); + break; } return preProcessor; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java new file mode 100644 index 00000000000..159dd94a4ed --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java @@ -0,0 +1,129 @@ +package org.apache.atlas.repository.store.graph.v2.preprocessor.contract; + +import org.apache.atlas.RequestContext; +import org.apache.atlas.authorize.AtlasAuthorizationUtils; +import org.apache.atlas.authorize.AtlasEntityAccessRequest; +import org.apache.atlas.authorize.AtlasPrivilege; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.model.instance.AtlasObjectId; +import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.resource.AbstractResourcePreProcessor; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Iterator; + +import static org.apache.atlas.repository.Constants.*; +import static org.apache.atlas.repository.Constants.ASSET_RELATION_ATTR; + +public abstract class AbstractContractPreProcessor implements PreProcessor { + private static final Logger LOG = LoggerFactory.getLogger(AbstractResourcePreProcessor.class); + + public final AtlasTypeRegistry typeRegistry; + public final EntityGraphRetriever entityRetriever; + public final AtlasGraph graph; + + + AbstractContractPreProcessor(AtlasGraph graph, AtlasTypeRegistry typeRegistry, + EntityGraphRetriever entityRetriever) { + this.graph = graph; + this.typeRegistry = typeRegistry; + this.entityRetriever = entityRetriever; + } + + void authorizeResourceUpdate(AtlasEntity resourceEntity, AtlasVertex ResourceVertex, String edgeLabel) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("authorizeResourceUpdate"); + + try { + AtlasEntityHeader assetEntity = null; + + AtlasObjectId asset = getAssetRelationAttr(resourceEntity); + if (asset != null) { + //Found linked asset in payload + AtlasVertex assetVertex = entityRetriever.getEntityVertex(asset); + assetEntity = entityRetriever.toAtlasEntityHeaderWithClassifications(assetVertex); + + } else { + //Check for linked asset in store + Iterator atlasVertexIterator = ResourceVertex.query() + .direction(AtlasEdgeDirection.IN) + .label(edgeLabel) + .has(STATE_PROPERTY_KEY, ACTIVE_STATE_VALUE) + .vertices() + .iterator(); + + if (atlasVertexIterator.hasNext()) { + //Found linked asset in store + AtlasVertex assetVertex = (AtlasVertex) atlasVertexIterator.next(); + assetEntity = entityRetriever.toAtlasEntityHeaderWithClassifications(assetVertex); + } + } + + if (assetEntity != null) { + //First authorize entity update access + verifyAssetAccess(assetEntity, AtlasPrivilege.ENTITY_UPDATE, resourceEntity, AtlasPrivilege.ENTITY_UPDATE); + } else { + //No linked asset to the Resource, check for resource update permission + verifyAccess(resourceEntity, AtlasPrivilege.ENTITY_UPDATE); + } + + } finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + } + + void authorizeResourceDelete(AtlasVertex resourceVertex) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("authorizeResourceDelete"); + + try { + AtlasEntity resourceEntity = entityRetriever.toAtlasEntity(resourceVertex); + + AtlasObjectId asset = getAssetRelationAttr(resourceEntity); + if (asset != null) { + AtlasEntityHeader assetEntity = entityRetriever.toAtlasEntityHeaderWithClassifications(asset.getGuid()); + verifyAssetAccess(assetEntity, AtlasPrivilege.ENTITY_UPDATE, resourceEntity, AtlasPrivilege.ENTITY_DELETE); + } else { + //No linked asset to the Resource, check for resource delete permission + verifyAccess(resourceEntity, AtlasPrivilege.ENTITY_DELETE); + } + } finally { + RequestContext.get().endMetricRecord(recorder); + } + } + + private AtlasObjectId getAssetRelationAttr(AtlasEntity entity) { + AtlasObjectId ret = null; + + if (entity.hasRelationshipAttribute(ASSET_RELATION_ATTR) && + entity.getRelationshipAttribute(ASSET_RELATION_ATTR) != null) { + ret = (AtlasObjectId) entity.getRelationshipAttribute(ASSET_RELATION_ATTR); + } + + return ret; + } + + private void verifyAssetAccess(AtlasEntityHeader asset, AtlasPrivilege assetPrivilege, + AtlasEntity resource, AtlasPrivilege resourcePrivilege) throws AtlasBaseException { + verifyAccess(asset, assetPrivilege); + verifyAccess(resource, resourcePrivilege); + } + + private void verifyAccess(AtlasEntity entity, AtlasPrivilege privilege) throws AtlasBaseException { + verifyAccess(new AtlasEntityHeader(entity), privilege); + } + + private void verifyAccess(AtlasEntityHeader entityHeader, AtlasPrivilege privilege) throws AtlasBaseException { + String errorMessage = privilege.name() + " entity: " + entityHeader.getTypeName(); + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, privilege, entityHeader), errorMessage); + } + + +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java new file mode 100644 index 00000000000..0b329ec47ef --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -0,0 +1,245 @@ +package org.apache.atlas.repository.store.graph.v2.preprocessor.contract; + +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.exc.InvalidFormatException; +import com.sun.xml.bind.v2.TODO; +import joptsimple.internal.Strings; +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.RequestContext; +import org.apache.atlas.authorize.AtlasAuthorizationUtils; +import org.apache.atlas.authorize.AtlasEntityAccessRequest; +import org.apache.atlas.authorize.AtlasPrivilege; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.featureflag.FeatureFlagStore; +import org.apache.atlas.model.TypeCategory; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.model.instance.AtlasStruct; +import org.apache.atlas.model.instance.EntityMutations; +import org.apache.atlas.repository.Constants; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.AtlasEntityStore; +import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; +import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; +import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.resource.AbstractResourcePreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.resource.ReadmePreProcessor; +import org.apache.atlas.type.AtlasEntityType; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.commons.lang.StringUtils; +import org.jetbrains.annotations.NotNull; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +//import org.pkl.config.java.Config; +//import org.pkl.config.java.ConfigEvaluator; +//import org.pkl.core.ModuleSource; +//import org.pkl.config.java.JavaType; + +//import org.pkl.codegen.java.JavaCodeGenerator; + +import com.fasterxml.jackson.databind.ObjectMapper; + + +import java.util.*; + +import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; +import static org.apache.atlas.repository.Constants.ASSET_README_EDGE_LABEL; +import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; +import static org.apache.atlas.repository.util.AccessControlUtils.*; +import static org.apache.atlas.repository.util.AccessControlUtils.getUUID; + +public class ContractPreProcessor extends AbstractContractPreProcessor { + private static final Logger LOG = LoggerFactory.getLogger(ReadmePreProcessor.class); + public static final String ATTR_CONTRACT = "contract"; + public static final String ATTR_CERTIFICATE_STATUS = "certificateStatus"; + private AtlasEntityStore entityStore; + + + public ContractPreProcessor(AtlasGraph graph, AtlasTypeRegistry typeRegistry, + EntityGraphRetriever entityRetriever, AtlasEntityStore entityStore) { + + super(graph, typeRegistry, entityRetriever); + this.entityStore = entityStore; + + } + + @Override + public void processAttributes(AtlasStruct entityStruct, EntityMutationContext context, EntityMutations.EntityOperation operation) throws AtlasBaseException { + AtlasEntity entity = (AtlasEntity) entityStruct; + switch (operation) { + case CREATE: + processCreateContract(entity, context); + break; + case UPDATE: + processUpdateContract(entity, context); + break; + } + + } + + private void processCreateContract(AtlasEntity entity, EntityMutationContext context) throws AtlasBaseException { + /* + Low-level Design + | Authorization + | Deserialization of the JSON + ---| Validation of spec + | Validation of contract + | Create Version + | Create Draft + ---| asset to contract sync + | Create Publish + ---| two-way sync of attribute + */ + + String contractString = (String) entity.getAttribute(ATTR_CONTRACT); + if (StringUtils.isEmpty(contractString)) { + throw new AtlasBaseException(BAD_REQUEST, "Please provide attribute " + ATTR_CONTRACT); + } + + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS); + DataContract contract; + try { + contract = objectMapper.readValue(contractString, DataContract.class); + } catch (Exception ex) { + ex.printStackTrace(); + throw new AtlasBaseException("Failed at this"); + } + + contractAttributeSync(entity, contract); + String contractQName = (String) entity.getAttribute(QUALIFIED_NAME); + validateAttribute(!contractQName.endsWith("/contract"), "Invalid qualifiedName for the contract."); + + String datasetQName = contractQName.substring(0, contractQName.lastIndexOf('/')); + ContractVersionUtils versionUtil = new ContractVersionUtils(entity, context, entityRetriever, typeRegistry, entityStore, graph); + AtlasEntity latestVersion = versionUtil.getLatestVersion(); + +// List vertexList = new ArrayList<>(); +// while (allContracts.hasNext()) { +// AtlasVertex v = allContracts.next(); +// vertexList.add(v); +// } +// RequestContext.get().endMetricRecord(metric); + + if (latestVersion != null) { + String qName = (String) latestVersion.getAttribute(QUALIFIED_NAME); + Integer latestVersionNumber = Integer.valueOf(qName.substring(qName.lastIndexOf("/V")+2)); + + entity.setAttribute(QUALIFIED_NAME, String.format("%s/version/V%s", contractQName, ++latestVersionNumber)); + + } else { + entity.setAttribute(QUALIFIED_NAME, String.format("%s/%s", contractQName, "version/V1")); + + } + + DataContract.Dataset dataset = contract.dataset; + AtlasEntityWithExtInfo associatedAsset = getAssociatedAsset(datasetQName, dataset.type.name()); + + if (contract.getStatus() == DataContract.STATUS.VERIFIED && + entity.getAttribute(ATTR_CERTIFICATE_STATUS).equals(DataContract.STATUS.VERIFIED.name())) { + datasetAttributeSync(associatedAsset.getEntity(), contract); + } + + +// versionUtil.createNewVersion(); + + } + + private void processUpdateContract(AtlasEntity entity, EntityMutationContext context) throws AtlasBaseException { + throw new AtlasBaseException("Can't update a specific version of contract"); + + + } + + private void contractAttributeSync(AtlasEntity entity, DataContract contract) { + // Sync certificateStatus + if (!Objects.equals(entity.getAttribute(ATTR_CERTIFICATE_STATUS), contract.getStatus().name())) { + /* + CertificateStatus | Status | Result + DRAFT VERIFIED cert -> VERIFIED > + VERIFIED DRAFT stat -> VERIFIED > + - DRAFT cert -> DRAFT + - VERIFIED cert -> VERIFIED > + DRAFT - stat -> DRAFT + VERIFIED - stat -> VERIFIED > + + */ + if (entity.getAttribute(ATTR_CERTIFICATE_STATUS) == DataContract.STATUS.VERIFIED.name()) { + contract.setStatus(DataContract.STATUS.VERIFIED); + } else if (contract.getStatus() == DataContract.STATUS.VERIFIED) { + entity.setAttribute(ATTR_CERTIFICATE_STATUS, DataContract.STATUS.VERIFIED.name()); + } else { + entity.setAttribute(ATTR_CERTIFICATE_STATUS, DataContract.STATUS.DRAFT); + contract.setStatus(DataContract.STATUS.DRAFT); + } + + } + + } + + private void datasetAttributeSync(AtlasEntity associatedAsset, DataContract contract) throws AtlasBaseException { + + DataContract.Dataset dataset = contract.dataset; + // Will implement dataset attribute sync from the contract attributes + + } + + private AtlasEntityWithExtInfo getLatestContract(String datasetQName) throws AtlasBaseException { + Map uniqAttributes = new HashMap<>(); + uniqAttributes.put(QUALIFIED_NAME, String.format("%s/%s", datasetQName, "contract/version/*")); + AtlasEntityType entityType = ensureEntityType(Constants.CONTRACT_ENTITY_TYPE); + + AtlasVertex entityVertex = AtlasGraphUtilsV2.getVertexByUniqueAttributes(graph, entityType, uniqAttributes); + + EntityGraphRetriever entityRetriever = new EntityGraphRetriever(graph, typeRegistry, true); + + AtlasEntityWithExtInfo ret = entityRetriever.toAtlasEntityWithExtInfo(entityVertex); + + if (ret == null) { + throw new AtlasBaseException(AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND, entityType.getTypeName(), + uniqAttributes.toString()); + } + return ret; + + } + private AtlasEntityWithExtInfo getAssociatedAsset(String qualifiedName, String typeName) throws AtlasBaseException { + Map uniqAttributes = new HashMap<>(); + uniqAttributes.put(QUALIFIED_NAME, qualifiedName); + + AtlasEntityType entityType = ensureEntityType(typeName); + + AtlasVertex entityVertex = AtlasGraphUtilsV2.getVertexByUniqueAttributes(graph, entityType, uniqAttributes); + + EntityGraphRetriever entityRetriever = new EntityGraphRetriever(graph, typeRegistry, true); + + AtlasEntityWithExtInfo ret = entityRetriever.toAtlasEntityWithExtInfo(entityVertex); + + if (ret == null) { + throw new AtlasBaseException(AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND, entityType.getTypeName(), + uniqAttributes.toString()); + } + return ret; + } + + private AtlasEntityType ensureEntityType(String typeName) throws AtlasBaseException { + AtlasEntityType ret = typeRegistry.getEntityTypeByName(typeName); + + if (ret == null) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), typeName); + } + + return ret; + } + + private static void validateAttribute(boolean isInvalid, String errorMessage) throws AtlasBaseException { + if (isInvalid) + throw new AtlasBaseException(BAD_REQUEST, errorMessage); + } +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java new file mode 100644 index 00000000000..e476986ff3c --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java @@ -0,0 +1,127 @@ +package org.apache.atlas.repository.store.graph.v2.preprocessor.contract; + +import org.apache.atlas.RequestContext; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasGraphQuery; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.AtlasEntityStore; +import org.apache.atlas.repository.store.graph.v2.*; +import org.apache.atlas.repository.store.graph.v2.preprocessor.ConnectionPreProcessor; +import org.apache.atlas.type.AtlasEntityType; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; + +import static org.apache.atlas.repository.Constants.*; +import static org.apache.atlas.repository.graph.AtlasGraphProvider.getGraphInstance; + + +public class ContractVersionUtils { + private static final Logger LOG = LoggerFactory.getLogger(ConnectionPreProcessor.class); + + private final EntityMutationContext context; + public final EntityGraphRetriever entityRetriever; + private final AtlasTypeRegistry atlasTypeRegistry; + private AtlasEntityStore entityStore; + + private AtlasEntity entity; + private AtlasEntity existingEntity; + public final AtlasGraph graph; + private List versionList; + + + + public ContractVersionUtils(AtlasEntity entity, EntityMutationContext context, EntityGraphRetriever entityRetriever, + AtlasTypeRegistry atlasTypeRegistry, AtlasEntityStore entityStore, AtlasGraph graph) { + this.context = context; + this.entityRetriever = entityRetriever; + this.atlasTypeRegistry = atlasTypeRegistry; + this.graph = graph; + this.entityStore = entityStore; + this.entity = entity; + } + + private void extractAllVersions() { + String entityQNamePrefix = (String) entity.getAttribute(QUALIFIED_NAME); + + AtlasEntityType entityType = atlasTypeRegistry.getEntityTypeByName("DataContract"); + Integer versionCounter = 1; + boolean found = true; + + List versionList = new ArrayList<>(); + + while (found) { + Map uniqAttributes = new HashMap<>(); + uniqAttributes.put(QUALIFIED_NAME, String.format("%s/version/V%s", entityQNamePrefix, versionCounter++)); + try { + AtlasVertex entityVertex = AtlasGraphUtilsV2.getVertexByUniqueAttributes(graph, entityType, uniqAttributes); + AtlasEntity entity = entityRetriever.toAtlasEntity(entityVertex); + + versionList.add(entity); + } catch (AtlasBaseException ex) { + found = false; + } + + } + this.versionList = versionList; + + } + + public AtlasEntity getLatestVersion() throws AtlasBaseException { + if (this.versionList == null) { + extractAllVersions(); + } + Collections.sort(this.versionList, (e1, e2) -> { + String e1QName = (String) e1.getAttribute(QUALIFIED_NAME); + String e2QName = (String) e2.getAttribute(QUALIFIED_NAME); + + return e2QName.compareTo(e1QName); + }); + if (this.versionList.isEmpty()) { + return null; + } + return this.versionList.get(0); + } + + + public Iterator getAllEntityVersions() { + String entityQNamePrefix = (String) entity.getAttribute(QUALIFIED_NAME); + AtlasEntityType entityType = atlasTypeRegistry.getEntityTypeByName("DataContract"); + +// AtlasEntityType entityType, String name + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("getAllEntityVersions"); + AtlasGraph graph = getGraphInstance(); + AtlasGraphQuery query = graph.query() + .has(ENTITY_TYPE_PROPERTY_KEY, entityType.getTypeName()) + .has(STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name()) + .has(entityType.getAllAttributes().get(QUALIFIED_NAME).getQualifiedName(), String.format("%s/version/V1", entityQNamePrefix)); + + + Iterator result = query.vertices().iterator(); + + RequestContext.get().endMetricRecord(metric); + return result; + } + + public void createNewVersion() throws AtlasBaseException { + AtlasVertex vertex = context.getVertex(entity.getGuid()); + AtlasEntity existingContractEntity = entityRetriever.toAtlasEntity(vertex); +// this.newEntity = new AtlasEntity(existingEntity); + this.existingEntity.setAttribute(QUALIFIED_NAME, null); + + try { + RequestContext.get().setSkipAuthorizationCheck(true); + EntityStream entityStream = new AtlasEntityStream(existingEntity); + entityStore.createOrUpdate(entityStream, false); + LOG.info("Created bootstrap policies for connection {}", existingEntity.getAttribute(QUALIFIED_NAME)); + } finally { + RequestContext.get().setSkipAuthorizationCheck(false); + } + + } +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java new file mode 100644 index 00000000000..63da3e909fe --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java @@ -0,0 +1,129 @@ +package org.apache.atlas.repository.store.graph.v2.preprocessor.contract; + +import java.lang.String; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSetter; +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.commons.lang.StringUtils; + + +@JsonIgnoreProperties(ignoreUnknown = true) +public class DataContract { + @JsonProperty(required = true) + public String kind; + public STATUS status; + @JsonProperty(value = "template_version", defaultValue = "0.0.1") + public String templateVersion; + public Dataset dataset; + public List columns; + + public STATUS getStatus() { + return status; + } + + @JsonSetter("status") + public void setStatus(STATUS status) { + this.status = status; + } + + public enum STATUS { + @JsonProperty("DRAFT") DRAFT, + @JsonProperty("VERIFIED") VERIFIED; + + public static STATUS from(String s) { + if(StringUtils.isEmpty(s)) { + return DRAFT; + } + + switch (s.toLowerCase()) { + case "draft": + return DRAFT; + + case "verified": + return VERIFIED; + + default: + return DRAFT; + } + } + } + + @JsonSetter("kind") + public void setKind(String kind) throws AtlasBaseException { + if (!"DataContract".equals(kind)) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "kind " + kind + " is inappropriate."); + } + this.kind = kind; + } + + public void setTemplateVersion(String templateVersion) { + if (!isSemVer(templateVersion)) { + throw new IllegalArgumentException("Invalid version syntax"); + } + this.templateVersion = templateVersion; + } + + private boolean isSemVer(String version) { + Pattern versionPattern = Pattern.compile("^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$"); + Matcher matcher = versionPattern.matcher(version); + return matcher.matches(); + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static final class Dataset { + public String name; + @JsonProperty(required = true) + public DATASET_TYPE type; + public String description; + + + @JsonSetter("type") + public void setType(DATASET_TYPE type) { + this.type = type; + } + + public enum DATASET_TYPE { + @JsonProperty("Table") Table, + @JsonProperty("View") View, + @JsonProperty("MaterialisedView") MaterialisedView; + + public static DATASET_TYPE from(String s) throws AtlasBaseException { + + switch (s.toLowerCase()) { + case "table": + return Table; + case "view": + return View; + case "materialisedview": + return MaterialisedView; + default: + throw new AtlasBaseException("dataset.type value not supported yet."); + } + } + } + + + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static final class Column { + public String name; + + public String description; + + public boolean is_primary; + + public String data_type; + + + } + + +} + From f5afe5dd0b727d57eade1aaaaae9810c41345a32 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Thu, 28 Mar 2024 13:39:36 +0530 Subject: [PATCH 016/316] superDomain issue resolved --- .../datamesh/DataProductPreProcessor.java | 18 ++++++++---------- .../datamesh/DomainPreProcessor.java | 18 +++++++----------- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index f83dff69138..d5b51125ee5 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -17,10 +17,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.*; @@ -133,17 +130,18 @@ private void setParent(AtlasEntity entity, EntityMutationContext context) throws AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("DataProductPreProcessor.setParent"); if (parentDomain == null) { Object relationshipAttribute = entity.getRelationshipAttribute(DATA_DOMAIN); + Set attributes = new HashSet<>(Arrays.asList(QUALIFIED_NAME, SUPER_DOMAIN_QN, PARENT_DOMAIN_QN, "__typeName")); if(relationshipAttribute instanceof AtlasObjectId){ AtlasObjectId objectId = (AtlasObjectId) relationshipAttribute; if (objectId != null) { if (StringUtils.isNotEmpty(objectId.getGuid())) { - AtlasVertex vertex = context.getVertex(objectId.getGuid()); + AtlasVertex vertex = entityRetriever.getEntityVertex(objectId.getGuid()); if (vertex == null) { - parentDomain = entityRetriever.toAtlasEntityHeader(objectId.getGuid()); + parentDomain = entityRetriever.toAtlasEntityHeader(objectId.getGuid(), attributes); } else { - parentDomain = entityRetriever.toAtlasEntityHeader(vertex); + parentDomain = entityRetriever.toAtlasEntityHeader(vertex, attributes); } } else if (MapUtils.isNotEmpty(objectId.getUniqueAttributes()) && @@ -156,12 +154,12 @@ private void setParent(AtlasEntity entity, EntityMutationContext context) throws else if(relationshipAttribute instanceof Map){ Map relationshipMap = (Map) relationshipAttribute; if (StringUtils.isNotEmpty((String) relationshipMap.get("guid"))) { - AtlasVertex vertex = context.getVertex((String) relationshipMap.get("guid")); + AtlasVertex vertex = entityRetriever.getEntityVertex((String) relationshipMap.get("guid")); if (vertex == null) { - parentDomain = entityRetriever.toAtlasEntityHeader((String) relationshipMap.get("guid")); + parentDomain = entityRetriever.toAtlasEntityHeader((String) relationshipMap.get("guid"), attributes); } else { - parentDomain = entityRetriever.toAtlasEntityHeader(vertex); + parentDomain = entityRetriever.toAtlasEntityHeader(vertex, attributes); } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index 2429b72bf15..d813c8c6847 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -41,11 +41,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.Iterator; -import java.util.List; -import java.util.Map; +import java.util.*; import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.graph.GraphHelper.getActiveChildrenVertices; @@ -268,21 +264,21 @@ private void setParent(AtlasEntity entity, EntityMutationContext context) throws AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("DomainPreProcessor.setParent"); if (parentDomain == null) { AtlasObjectId objectId = (AtlasObjectId) entity.getRelationshipAttribute(PARENT_DOMAIN); + Set attributes = new HashSet<>(Arrays.asList(QUALIFIED_NAME, SUPER_DOMAIN_QN, PARENT_DOMAIN_QN, "__typeName")); if (objectId != null) { if (StringUtils.isNotEmpty(objectId.getGuid())) { - AtlasVertex vertex = context.getVertex(objectId.getGuid()); + AtlasVertex vertex = entityRetriever.getEntityVertex(objectId.getGuid()); if (vertex == null) { - parentDomain = entityRetriever.toAtlasEntityHeader(objectId.getGuid()); + parentDomain = entityRetriever.toAtlasEntityHeader(objectId.getGuid(), attributes); } else { - parentDomain = entityRetriever.toAtlasEntityHeader(vertex); + parentDomain = entityRetriever.toAtlasEntityHeader(vertex, attributes); } - } else if (MapUtils.isNotEmpty(objectId.getUniqueAttributes()) && StringUtils.isNotEmpty((String) objectId.getUniqueAttributes().get(QUALIFIED_NAME))) { - parentDomain = new AtlasEntityHeader(objectId.getTypeName(), objectId.getUniqueAttributes()); - + AtlasVertex parentDomainVertex = entityRetriever.getEntityVertex(objectId); + parentDomain = entityRetriever.toAtlasEntityHeader(parentDomainVertex); } } } From 43bd8fae18712984bce4a7562eb78e745bc0e020 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Thu, 28 Mar 2024 13:46:58 +0530 Subject: [PATCH 017/316] parentdomain set approach changed --- .../v2/preprocessor/datamesh/DataProductPreProcessor.java | 3 ++- .../graph/v2/preprocessor/datamesh/DomainPreProcessor.java | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index d5b51125ee5..c806d1f179f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -146,7 +146,8 @@ private void setParent(AtlasEntity entity, EntityMutationContext context) throws } else if (MapUtils.isNotEmpty(objectId.getUniqueAttributes()) && StringUtils.isNotEmpty((String) objectId.getUniqueAttributes().get(QUALIFIED_NAME))) { - parentDomain = new AtlasEntityHeader(objectId.getTypeName(), objectId.getUniqueAttributes()); + AtlasVertex parentDomainVertex = entityRetriever.getEntityVertex(objectId); + parentDomain = entityRetriever.toAtlasEntityHeader(parentDomainVertex, attributes); } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index d813c8c6847..c5b26ad8ce8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -278,7 +278,7 @@ private void setParent(AtlasEntity entity, EntityMutationContext context) throws } else if (MapUtils.isNotEmpty(objectId.getUniqueAttributes()) && StringUtils.isNotEmpty((String) objectId.getUniqueAttributes().get(QUALIFIED_NAME))) { AtlasVertex parentDomainVertex = entityRetriever.getEntityVertex(objectId); - parentDomain = entityRetriever.toAtlasEntityHeader(parentDomainVertex); + parentDomain = entityRetriever.toAtlasEntityHeader(parentDomainVertex, attributes); } } } From 2cfd40b51b12a7628b2d829c1adfe29676b2693b Mon Sep 17 00:00:00 2001 From: Rajat Movaliya Date: Fri, 29 Mar 2024 17:24:34 +0530 Subject: [PATCH 018/316] changes to make it work --- .../org/apache/atlas/notification/AbstractNotification.java | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/notification/src/main/java/org/apache/atlas/notification/AbstractNotification.java b/notification/src/main/java/org/apache/atlas/notification/AbstractNotification.java index cca4cb81d89..bed35932a11 100644 --- a/notification/src/main/java/org/apache/atlas/notification/AbstractNotification.java +++ b/notification/src/main/java/org/apache/atlas/notification/AbstractNotification.java @@ -96,7 +96,7 @@ public void send(NotificationType type, List messages, MessageSource sour createNotificationMessages(messages.get(index), strMessages, source); } - sendInternal(type, strMessages); +// sendInternal(type, strMessages); } @Override diff --git a/pom.xml b/pom.xml index 1cc9aa70dc9..3abc39041b7 100644 --- a/pom.xml +++ b/pom.xml @@ -717,7 +717,7 @@ 4.4.13 2.12.4 2.12.4 - 0.6.03 + 0.6.04 0.5.3 1 3.1.0 From 379d39453788ce567944ab317474acf7265da8a7 Mon Sep 17 00:00:00 2001 From: Rajat Movaliya Date: Tue, 2 Apr 2024 18:31:59 +0530 Subject: [PATCH 019/316] feat: add node depth in lineage --- .../model/instance/AtlasEntityHeader.java | 10 ++++++++++ .../atlas/discovery/EntityLineageService.java | 18 ++++++++++-------- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java index be2819d7f20..f5f060349dd 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java @@ -71,8 +71,10 @@ public class AtlasEntityHeader extends AtlasStruct implements Serializable { private Date createTime = null; private Date updateTime = null; private String deleteHandler = null; + private Integer depth = null; private Map collapse = null; + public AtlasEntityHeader() { this(null, null); } @@ -154,6 +156,14 @@ public void setGuid(String guid) { this.guid = guid; } + public Integer getDepth() { + return depth; + } + + public void setDepth(Integer depth) { + this.depth = depth; + } + public AtlasEntity.Status getStatus() { return status; } diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index 6927b1d1c4f..b64739657b7 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -336,7 +336,7 @@ private void traverseEdgesOnDemand(Iterator processEdges, boolean isI if (incrementAndCheckIfRelationsLimitReached(processEdge, isInputEdge, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, direction)) { break; } else { - addEdgeToResult(processEdge, ret, atlasLineageOnDemandContext); + addEdgeToResult(processEdge, ret, atlasLineageOnDemandContext, depth); } String inGuid = AtlasGraphUtilsV2.getIdFromVertex(datasetVertex); @@ -385,7 +385,7 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i else continue; } else { - addEdgeToResult(incomingEdge, ret, atlasLineageOnDemandContext); + addEdgeToResult(incomingEdge, ret, atlasLineageOnDemandContext, depth); } AtlasPerfMetrics.MetricRecorder traverseEdgesOnDemandGetEdgesOut = RequestContext.get().startMetricRecord("traverseEdgesOnDemandGetEdgesOut"); @@ -413,7 +413,7 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i else continue; } else { - addEdgeToResult(outgoingEdge, ret, atlasLineageOnDemandContext); + addEdgeToResult(outgoingEdge, ret, atlasLineageOnDemandContext, depth); entitiesTraversed.incrementAndGet(); if (isEntityTraversalLimitReached(entitiesTraversed)) setEntityLimitReachedFlag(isInput, ret); @@ -874,9 +874,9 @@ private void addEdgeToResult(AtlasEdge edge, AtlasLineageInfo lineageInfo, } } - private void addEdgeToResult(AtlasEdge edge, AtlasLineageOnDemandInfo lineageInfo, AtlasLineageOnDemandContext atlasLineageOnDemandContext) throws AtlasBaseException { + private void addEdgeToResult(AtlasEdge edge, AtlasLineageOnDemandInfo lineageInfo, AtlasLineageOnDemandContext atlasLineageOnDemandContext, int depth) throws AtlasBaseException { if (!lineageContainsVisitedEdgeV2(lineageInfo, edge)) { - processEdge(edge, lineageInfo, atlasLineageOnDemandContext); + processEdge(edge, lineageInfo, atlasLineageOnDemandContext, depth); } } @@ -1446,11 +1446,11 @@ private void processEdge(final AtlasEdge edge, final Map entities, final Set relations, final Set visitedEdges, final Set attributes) throws AtlasBaseException { + private void processEdge(final AtlasEdge edge, final Map entities, final Set relations, final Set visitedEdges, final Set attributes, int depth) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processEdge"); AtlasVertex inVertex = edge.getInVertex(); AtlasVertex outVertex = edge.getOutVertex(); @@ -1462,10 +1462,12 @@ private void processEdge(final AtlasEdge edge, final Map Date: Wed, 3 Apr 2024 10:15:52 +0530 Subject: [PATCH 020/316] Refactore contract code and release --- .github/workflows/maven.yml | 1 + .../apache/atlas/repository/Constants.java | 1 + .../store/graph/v2/AtlasEntityStoreV2.java | 2 +- .../AbstractContractPreProcessor.java | 120 ++++---- .../contract/ContractPreProcessor.java | 265 +++++++++--------- .../contract/ContractVersionUtils.java | 50 ++-- .../preprocessor/contract/DataContract.java | 30 ++ 7 files changed, 258 insertions(+), 211 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 0977cb36a25..39703359df6 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,6 +26,7 @@ on: - development - master - lineageondemand + - data-contract jobs: build: diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 9fdb249fcc1..925d0a64769 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -399,6 +399,7 @@ public enum SupportedFileExtensions { XLSX, XLS, CSV } public static final String ATTR_STARRED_DETAILS_LIST = "starredDetailsList"; public static final String ATTR_ASSET_STARRED_BY = "assetStarredBy"; public static final String ATTR_ASSET_STARRED_AT = "assetStarredAt"; + public static final String ATTR_CERTIFICATE_STATUS = "certificateStatus"; public static final String STRUCT_STARRED_DETAILS = "StarredDetails"; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 4daf9128dae..02dd3965b87 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -1838,7 +1838,7 @@ public PreProcessor getPreProcessor(String typeName) { break; case CONTRACT_ENTITY_TYPE: - preProcessor = new ContractPreProcessor(graph, typeRegistry, entityRetriever, this); + preProcessor = new ContractPreProcessor(graph, typeRegistry, entityRetriever, this, entityGraphMapper); break; } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java index 159dd94a4ed..dd2e558156b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java @@ -5,27 +5,29 @@ import org.apache.atlas.authorize.AtlasEntityAccessRequest; import org.apache.atlas.authorize.AtlasPrivilege; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.TypeCategory; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntityHeader; -import org.apache.atlas.model.instance.AtlasObjectId; -import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; -import org.apache.atlas.repository.store.graph.v2.preprocessor.resource.AbstractResourcePreProcessor; +import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Iterator; +import java.util.HashMap; +import java.util.Map; +import static org.apache.atlas.AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND; +import static org.apache.atlas.AtlasErrorCode.TYPE_NAME_INVALID; import static org.apache.atlas.repository.Constants.*; -import static org.apache.atlas.repository.Constants.ASSET_RELATION_ATTR; public abstract class AbstractContractPreProcessor implements PreProcessor { - private static final Logger LOG = LoggerFactory.getLogger(AbstractResourcePreProcessor.class); + private static final Logger LOG = LoggerFactory.getLogger(AbstractContractPreProcessor.class); public final AtlasTypeRegistry typeRegistry; public final EntityGraphRetriever entityRetriever; @@ -39,81 +41,40 @@ public abstract class AbstractContractPreProcessor implements PreProcessor { this.entityRetriever = entityRetriever; } - void authorizeResourceUpdate(AtlasEntity resourceEntity, AtlasVertex ResourceVertex, String edgeLabel) throws AtlasBaseException { - AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("authorizeResourceUpdate"); - + void authorizeContractCreateOrUpdate(AtlasEntity contractEntity, AtlasEntity.AtlasEntityWithExtInfo associatedAsset) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("authorizeContractUpdate"); try { - AtlasEntityHeader assetEntity = null; - - AtlasObjectId asset = getAssetRelationAttr(resourceEntity); - if (asset != null) { - //Found linked asset in payload - AtlasVertex assetVertex = entityRetriever.getEntityVertex(asset); - assetEntity = entityRetriever.toAtlasEntityHeaderWithClassifications(assetVertex); - - } else { - //Check for linked asset in store - Iterator atlasVertexIterator = ResourceVertex.query() - .direction(AtlasEdgeDirection.IN) - .label(edgeLabel) - .has(STATE_PROPERTY_KEY, ACTIVE_STATE_VALUE) - .vertices() - .iterator(); - - if (atlasVertexIterator.hasNext()) { - //Found linked asset in store - AtlasVertex assetVertex = (AtlasVertex) atlasVertexIterator.next(); - assetEntity = entityRetriever.toAtlasEntityHeaderWithClassifications(assetVertex); - } - } - - if (assetEntity != null) { - //First authorize entity update access - verifyAssetAccess(assetEntity, AtlasPrivilege.ENTITY_UPDATE, resourceEntity, AtlasPrivilege.ENTITY_UPDATE); - } else { - //No linked asset to the Resource, check for resource update permission - verifyAccess(resourceEntity, AtlasPrivilege.ENTITY_UPDATE); - } + AtlasEntityHeader entityHeader = new AtlasEntityHeader(associatedAsset.getEntity()); + + //First authorize entity update access + verifyAssetAccess(entityHeader, AtlasPrivilege.ENTITY_UPDATE, contractEntity, AtlasPrivilege.ENTITY_UPDATE); } finally { RequestContext.get().endMetricRecord(metricRecorder); } } - void authorizeResourceDelete(AtlasVertex resourceVertex) throws AtlasBaseException { - AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("authorizeResourceDelete"); + + void authorizeContractDelete(AtlasVertex contractVertex, String typeName) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("authorizeContractDelete"); try { - AtlasEntity resourceEntity = entityRetriever.toAtlasEntity(resourceVertex); - - AtlasObjectId asset = getAssetRelationAttr(resourceEntity); - if (asset != null) { - AtlasEntityHeader assetEntity = entityRetriever.toAtlasEntityHeaderWithClassifications(asset.getGuid()); - verifyAssetAccess(assetEntity, AtlasPrivilege.ENTITY_UPDATE, resourceEntity, AtlasPrivilege.ENTITY_DELETE); - } else { - //No linked asset to the Resource, check for resource delete permission - verifyAccess(resourceEntity, AtlasPrivilege.ENTITY_DELETE); - } + AtlasEntity contractEntity = entityRetriever.toAtlasEntity(contractVertex); + String contractQName = contractEntity.getAttribute(QUALIFIED_NAME).toString(); + AtlasEntity.AtlasEntityWithExtInfo assetEntity = getAssociatedAsset(contractQName, typeName); + AtlasEntityHeader entityHeader = new AtlasEntityHeader(assetEntity.getEntity()); + + verifyAssetAccess(entityHeader, AtlasPrivilege.ENTITY_UPDATE, contractEntity, AtlasPrivilege.ENTITY_DELETE); } finally { RequestContext.get().endMetricRecord(recorder); } } - private AtlasObjectId getAssetRelationAttr(AtlasEntity entity) { - AtlasObjectId ret = null; - - if (entity.hasRelationshipAttribute(ASSET_RELATION_ATTR) && - entity.getRelationshipAttribute(ASSET_RELATION_ATTR) != null) { - ret = (AtlasObjectId) entity.getRelationshipAttribute(ASSET_RELATION_ATTR); - } - - return ret; - } private void verifyAssetAccess(AtlasEntityHeader asset, AtlasPrivilege assetPrivilege, - AtlasEntity resource, AtlasPrivilege resourcePrivilege) throws AtlasBaseException { + AtlasEntity contract, AtlasPrivilege contractPrivilege) throws AtlasBaseException { verifyAccess(asset, assetPrivilege); - verifyAccess(resource, resourcePrivilege); + verifyAccess(contract, contractPrivilege); } private void verifyAccess(AtlasEntity entity, AtlasPrivilege privilege) throws AtlasBaseException { @@ -125,5 +86,36 @@ private void verifyAccess(AtlasEntityHeader entityHeader, AtlasPrivilege privile AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, privilege, entityHeader), errorMessage); } + AtlasEntity.AtlasEntityWithExtInfo getAssociatedAsset(String contractQName, String typeName) throws AtlasBaseException { + String datasetQName = contractQName.substring(0, contractQName.lastIndexOf('/')); + + Map uniqAttributes = new HashMap<>(); + uniqAttributes.put(QUALIFIED_NAME, datasetQName); + + AtlasEntityType entityType = ensureEntityType(typeName); + + AtlasVertex entityVertex = AtlasGraphUtilsV2.getVertexByUniqueAttributes(graph, entityType, uniqAttributes); + + EntityGraphRetriever entityRetriever = new EntityGraphRetriever(graph, typeRegistry, true); + + AtlasEntity.AtlasEntityWithExtInfo ret = entityRetriever.toAtlasEntityWithExtInfo(entityVertex); + + if (ret == null) { + throw new AtlasBaseException(INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND, entityType.getTypeName(), + uniqAttributes.toString()); + } + return ret; + } + + AtlasEntityType ensureEntityType(String typeName) throws AtlasBaseException { + AtlasEntityType ret = typeRegistry.getEntityTypeByName(typeName); + + if (ret == null) { + throw new AtlasBaseException(TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), typeName); + } + + return ret; + } + } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index 0b329ec47ef..aecca4ea9d8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -1,73 +1,48 @@ package org.apache.atlas.repository.store.graph.v2.preprocessor.contract; -import com.fasterxml.jackson.databind.MapperFeature; -import com.fasterxml.jackson.databind.exc.InvalidFormatException; -import com.sun.xml.bind.v2.TODO; -import joptsimple.internal.Strings; -import org.apache.atlas.AtlasErrorCode; -import org.apache.atlas.RequestContext; -import org.apache.atlas.authorize.AtlasAuthorizationUtils; -import org.apache.atlas.authorize.AtlasEntityAccessRequest; -import org.apache.atlas.authorize.AtlasPrivilege; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.featureflag.FeatureFlagStore; -import org.apache.atlas.model.TypeCategory; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; -import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasStruct; import org.apache.atlas.model.instance.EntityMutations; -import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.AtlasEntityStore; -import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; -import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; -import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; -import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; -import org.apache.atlas.repository.store.graph.v2.preprocessor.resource.AbstractResourcePreProcessor; -import org.apache.atlas.repository.store.graph.v2.preprocessor.resource.ReadmePreProcessor; +import org.apache.atlas.repository.store.graph.v2.*; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; -import org.apache.commons.lang.StringUtils; -import org.jetbrains.annotations.NotNull; -import org.json.simple.JSONObject; -import org.json.simple.parser.JSONParser; -import org.json.simple.parser.ParseException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -//import org.pkl.config.java.Config; -//import org.pkl.config.java.ConfigEvaluator; -//import org.pkl.core.ModuleSource; -//import org.pkl.config.java.JavaType; - -//import org.pkl.codegen.java.JavaCodeGenerator; - import com.fasterxml.jackson.databind.ObjectMapper; - import java.util.*; -import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; -import static org.apache.atlas.repository.Constants.ASSET_README_EDGE_LABEL; +import static org.apache.atlas.AtlasErrorCode.*; +import static org.apache.atlas.repository.Constants.ATTR_CERTIFICATE_STATUS; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; -import static org.apache.atlas.repository.util.AccessControlUtils.*; -import static org.apache.atlas.repository.util.AccessControlUtils.getUUID; public class ContractPreProcessor extends AbstractContractPreProcessor { - private static final Logger LOG = LoggerFactory.getLogger(ReadmePreProcessor.class); - public static final String ATTR_CONTRACT = "contract"; - public static final String ATTR_CERTIFICATE_STATUS = "certificateStatus"; - private AtlasEntityStore entityStore; + private static final Logger LOG = LoggerFactory.getLogger(ContractPreProcessor.class); + public static final String ATTR_CONTRACT = "contract"; + public static final String CONTRACT_QUALIFIED_NAME_SUFFIX = "contract"; + public static final String VERSION_PREFIX = "version"; + public static final String CONTRACT_ATTR_STATUS = "status"; + private final AtlasEntityStore entityStore; + private final EntityGraphMapper entityGraphMapper; public ContractPreProcessor(AtlasGraph graph, AtlasTypeRegistry typeRegistry, - EntityGraphRetriever entityRetriever, AtlasEntityStore entityStore) { + EntityGraphRetriever entityRetriever, AtlasEntityStore entityStore, EntityGraphMapper entityGraphMapper) { super(graph, typeRegistry, entityRetriever); this.entityStore = entityStore; - + this.entityGraphMapper = entityGraphMapper; } @Override @@ -79,7 +54,49 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co break; case UPDATE: processUpdateContract(entity, context); - break; + } + + } + + private void processUpdateContract(AtlasEntity entity, EntityMutationContext context) throws AtlasBaseException { + String contractString = (String) entity.getAttribute(ATTR_CONTRACT); + AtlasVertex vertex = context.getVertex(entity.getGuid()); + AtlasEntity existingContractEntity = entityRetriever.toAtlasEntity(vertex); + + if (!isEqualContract(contractString, (String) existingContractEntity.getAttribute(ATTR_CONTRACT))) { + // Update the same asset(entity) + throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Can't update a specific version of contract"); + } + // Add cases for update in status field and certificateStatus + } + + private List getDiffAttributes(EntityMutationContext context, AtlasEntity entity, AtlasEntity latestExistingVersion) throws AtlasBaseException { + AtlasEntityComparator entityComparator = new AtlasEntityComparator(typeRegistry, entityRetriever, context.getGuidAssignments(), true, true); + AtlasEntityComparator.AtlasEntityDiffResult diffResult = entityComparator.getDiffResult(entity, latestExistingVersion, false); + List attributesSet = new ArrayList<>(); + + if (diffResult.hasDifference()) { + for (Map.Entry entry : diffResult.getDiffEntity().getAttributes().entrySet()) { + if (!entry.getKey().equals(QUALIFIED_NAME)) { + attributesSet.add(entry.getKey()); + } + } + } + return attributesSet; + } + + private boolean isEqualContract(String firstNode, String secondNode) throws AtlasBaseException { + ObjectMapper mapper = new ObjectMapper(); + try { + JsonNode actualObj1 = mapper.readTree(firstNode); + JsonNode actualObj2 = mapper.readTree(secondNode); + //Ignore status field change + ((ObjectNode) actualObj1).remove(CONTRACT_ATTR_STATUS); + ((ObjectNode) actualObj2).remove(CONTRACT_ATTR_STATUS); + + return actualObj1.equals(actualObj2); + } catch (JsonProcessingException e) { + throw new AtlasBaseException(JSON_ERROR, e.getMessage()); } } @@ -98,63 +115,104 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con ---| two-way sync of attribute */ + String contractQName = (String) entity.getAttribute(QUALIFIED_NAME); String contractString = (String) entity.getAttribute(ATTR_CONTRACT); - if (StringUtils.isEmpty(contractString)) { - throw new AtlasBaseException(BAD_REQUEST, "Please provide attribute " + ATTR_CONTRACT); - } + DataContract contract = DataContract.deserialize(contractString); + DataContract.Dataset dataset = contract.dataset; + AtlasEntityWithExtInfo associatedAsset = getAssociatedAsset(contractQName, dataset.type.name()); - ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS); - DataContract contract; - try { - contract = objectMapper.readValue(contractString, DataContract.class); - } catch (Exception ex) { - ex.printStackTrace(); - throw new AtlasBaseException("Failed at this"); - } + authorizeContractCreateOrUpdate(entity, associatedAsset); contractAttributeSync(entity, contract); - String contractQName = (String) entity.getAttribute(QUALIFIED_NAME); - validateAttribute(!contractQName.endsWith("/contract"), "Invalid qualifiedName for the contract."); - - String datasetQName = contractQName.substring(0, contractQName.lastIndexOf('/')); - ContractVersionUtils versionUtil = new ContractVersionUtils(entity, context, entityRetriever, typeRegistry, entityStore, graph); - AtlasEntity latestVersion = versionUtil.getLatestVersion(); + contractString = updateContract(contract, contractString); + entity.setAttribute(ATTR_CONTRACT, contractString); -// List vertexList = new ArrayList<>(); -// while (allContracts.hasNext()) { -// AtlasVertex v = allContracts.next(); -// vertexList.add(v); -// } -// RequestContext.get().endMetricRecord(metric); + validateAttribute(!contractQName.endsWith(String.format("/%s", CONTRACT_QUALIFIED_NAME_SUFFIX)), "Invalid qualifiedName for the contract."); - if (latestVersion != null) { - String qName = (String) latestVersion.getAttribute(QUALIFIED_NAME); - Integer latestVersionNumber = Integer.valueOf(qName.substring(qName.lastIndexOf("/V")+2)); + ContractVersionUtils versionUtil = new ContractVersionUtils(entity, context, entityRetriever, typeRegistry, entityStore, graph); + AtlasEntity latestExistingVersion = versionUtil.getLatestVersion(); + + if (latestExistingVersion != null) { + // Contract already exist + String qName = (String) latestExistingVersion.getAttribute(QUALIFIED_NAME); + Integer latestVersionNumber = Integer.valueOf(qName.substring(qName.lastIndexOf("/V") + 2)); + List attributes = getDiffAttributes(context, entity, latestExistingVersion); + if (attributes.isEmpty()) { + context.getCreatedEntities().remove(entity); + Set guids = new HashSet<>(); + guids.add(entity.getGuid()); + entityStore.purgeByIds(guids); + return; + } - entity.setAttribute(QUALIFIED_NAME, String.format("%s/version/V%s", contractQName, ++latestVersionNumber)); + if (attributes.size() == 1 && attributes.contains(ATTR_CERTIFICATE_STATUS)) { + if (Objects.equals(entity.getAttribute(ATTR_CERTIFICATE_STATUS).toString(), DataContract.STATUS.VERIFIED.name())) { + //update existing entity + updateExistingVersion(context, entity, latestExistingVersion); + } + // Contract status changed, either to draft or verified + return; + } else if (attributes.contains(ATTR_CONTRACT)) { + //Contract is changed + if (isEqualContract(contractString, (String) latestExistingVersion.getAttribute(ATTR_CONTRACT))) { + // Update the same asset(entity) + updateExistingVersion(context, entity, latestExistingVersion); + + } else { + // Create New version of entity + entity.setAttribute(QUALIFIED_NAME, String.format("%s/%s/V%s", contractQName, VERSION_PREFIX, ++latestVersionNumber)); + + } + + return; + } } else { - entity.setAttribute(QUALIFIED_NAME, String.format("%s/%s", contractQName, "version/V1")); + // Create new contract + entity.setAttribute(QUALIFIED_NAME, String.format("%s/%s/%s", contractQName, VERSION_PREFIX, "V1")); } - DataContract.Dataset dataset = contract.dataset; - AtlasEntityWithExtInfo associatedAsset = getAssociatedAsset(datasetQName, dataset.type.name()); - + entity.setAttribute("assetQualifiedName", associatedAsset.getEntity().getAttribute(QUALIFIED_NAME)); if (contract.getStatus() == DataContract.STATUS.VERIFIED && entity.getAttribute(ATTR_CERTIFICATE_STATUS).equals(DataContract.STATUS.VERIFIED.name())) { datasetAttributeSync(associatedAsset.getEntity(), contract); } + } -// versionUtil.createNewVersion(); + private String updateContract(DataContract contract, String contractString) throws AtlasBaseException { + ObjectMapper objectMapper = new ObjectMapper(); + try { + String contractObjString = DataContract.serialize(contract); + Map mapJson = objectMapper.readValue(contractString, new TypeReference>() { + }); + Map mapTargetJson = objectMapper.readValue(contractObjString, new TypeReference>() { + }); + + for (Map.Entry keyVal : mapTargetJson.entrySet()) { + mapJson.replace(keyVal.getKey(), keyVal.getValue()); + } + return objectMapper.writeValueAsString(mapJson); + } catch (JsonProcessingException ex) { + throw new AtlasBaseException(JSON_ERROR, ex.getMessage()); + } } - private void processUpdateContract(AtlasEntity entity, EntityMutationContext context) throws AtlasBaseException { - throw new AtlasBaseException("Can't update a specific version of contract"); + private void updateExistingVersion(EntityMutationContext context, AtlasEntity entity, AtlasEntity latestExistingVersion) throws AtlasBaseException { + context.getCreatedEntities().remove(entity); + Set guids = new HashSet<>(); + guids.add(entity.getGuid()); + entityStore.purgeByIds(guids); + entity.setAttribute(QUALIFIED_NAME, latestExistingVersion.getAttribute(QUALIFIED_NAME)); + entity.setGuid(latestExistingVersion.getGuid()); + + AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(entity.getGuid()); + + AtlasEntityType entityType = ensureEntityType(entity.getTypeName()); + context.addUpdated(entity.getGuid(), entity, entityType, vertex); } @@ -171,9 +229,9 @@ private void contractAttributeSync(AtlasEntity entity, DataContract contract) { VERIFIED - stat -> VERIFIED > */ - if (entity.getAttribute(ATTR_CERTIFICATE_STATUS) == DataContract.STATUS.VERIFIED.name()) { + if (Objects.equals(entity.getAttribute(ATTR_CERTIFICATE_STATUS), DataContract.STATUS.VERIFIED.name())) { contract.setStatus(DataContract.STATUS.VERIFIED); - } else if (contract.getStatus() == DataContract.STATUS.VERIFIED) { + } else if (Objects.equals(contract.getStatus(), DataContract.STATUS.VERIFIED)) { entity.setAttribute(ATTR_CERTIFICATE_STATUS, DataContract.STATUS.VERIFIED.name()); } else { entity.setAttribute(ATTR_CERTIFICATE_STATUS, DataContract.STATUS.DRAFT); @@ -191,53 +249,6 @@ private void datasetAttributeSync(AtlasEntity associatedAsset, DataContract cont } - private AtlasEntityWithExtInfo getLatestContract(String datasetQName) throws AtlasBaseException { - Map uniqAttributes = new HashMap<>(); - uniqAttributes.put(QUALIFIED_NAME, String.format("%s/%s", datasetQName, "contract/version/*")); - AtlasEntityType entityType = ensureEntityType(Constants.CONTRACT_ENTITY_TYPE); - - AtlasVertex entityVertex = AtlasGraphUtilsV2.getVertexByUniqueAttributes(graph, entityType, uniqAttributes); - - EntityGraphRetriever entityRetriever = new EntityGraphRetriever(graph, typeRegistry, true); - - AtlasEntityWithExtInfo ret = entityRetriever.toAtlasEntityWithExtInfo(entityVertex); - - if (ret == null) { - throw new AtlasBaseException(AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND, entityType.getTypeName(), - uniqAttributes.toString()); - } - return ret; - - } - private AtlasEntityWithExtInfo getAssociatedAsset(String qualifiedName, String typeName) throws AtlasBaseException { - Map uniqAttributes = new HashMap<>(); - uniqAttributes.put(QUALIFIED_NAME, qualifiedName); - - AtlasEntityType entityType = ensureEntityType(typeName); - - AtlasVertex entityVertex = AtlasGraphUtilsV2.getVertexByUniqueAttributes(graph, entityType, uniqAttributes); - - EntityGraphRetriever entityRetriever = new EntityGraphRetriever(graph, typeRegistry, true); - - AtlasEntityWithExtInfo ret = entityRetriever.toAtlasEntityWithExtInfo(entityVertex); - - if (ret == null) { - throw new AtlasBaseException(AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND, entityType.getTypeName(), - uniqAttributes.toString()); - } - return ret; - } - - private AtlasEntityType ensureEntityType(String typeName) throws AtlasBaseException { - AtlasEntityType ret = typeRegistry.getEntityTypeByName(typeName); - - if (ret == null) { - throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), typeName); - } - - return ret; - } - private static void validateAttribute(boolean isInvalid, String errorMessage) throws AtlasBaseException { if (isInvalid) throw new AtlasBaseException(BAD_REQUEST, errorMessage); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java index e476986ff3c..8d27a10bc1b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java @@ -1,5 +1,6 @@ package org.apache.atlas.repository.store.graph.v2.preprocessor.contract; +import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.RequestContext; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.AtlasEntity; @@ -30,7 +31,7 @@ public class ContractVersionUtils { private AtlasEntityStore entityStore; private AtlasEntity entity; - private AtlasEntity existingEntity; + public final AtlasGraph graph; private List versionList; @@ -47,7 +48,8 @@ public ContractVersionUtils(AtlasEntity entity, EntityMutationContext context, E } private void extractAllVersions() { - String entityQNamePrefix = (String) entity.getAttribute(QUALIFIED_NAME); + String contractQName = (String) entity.getAttribute(QUALIFIED_NAME); + String datasetQName = contractQName.substring(0, contractQName.lastIndexOf("/contract")); AtlasEntityType entityType = atlasTypeRegistry.getEntityTypeByName("DataContract"); Integer versionCounter = 1; @@ -57,10 +59,20 @@ private void extractAllVersions() { while (found) { Map uniqAttributes = new HashMap<>(); - uniqAttributes.put(QUALIFIED_NAME, String.format("%s/version/V%s", entityQNamePrefix, versionCounter++)); + uniqAttributes.put(QUALIFIED_NAME, String.format("%s/contract/version/V%s", datasetQName, versionCounter++)); try { AtlasVertex entityVertex = AtlasGraphUtilsV2.getVertexByUniqueAttributes(graph, entityType, uniqAttributes); AtlasEntity entity = entityRetriever.toAtlasEntity(entityVertex); +// +// EntityGraphRetriever entityRetriever = new EntityGraphRetriever(graph, typeRegistry, true); +// +// AtlasEntity.AtlasEntityWithExtInfo ret = entityRetriever.toAtlasEntityWithExtInfo(entityVertex); +// +// if (ret == null) { +// throw new AtlasBaseException(AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND, entityType.getTypeName(), +// uniqAttributes.toString()); +// } +// return ret; versionList.add(entity); } catch (AtlasBaseException ex) { @@ -108,20 +120,20 @@ public Iterator getAllEntityVersions() { return result; } - public void createNewVersion() throws AtlasBaseException { - AtlasVertex vertex = context.getVertex(entity.getGuid()); - AtlasEntity existingContractEntity = entityRetriever.toAtlasEntity(vertex); -// this.newEntity = new AtlasEntity(existingEntity); - this.existingEntity.setAttribute(QUALIFIED_NAME, null); - - try { - RequestContext.get().setSkipAuthorizationCheck(true); - EntityStream entityStream = new AtlasEntityStream(existingEntity); - entityStore.createOrUpdate(entityStream, false); - LOG.info("Created bootstrap policies for connection {}", existingEntity.getAttribute(QUALIFIED_NAME)); - } finally { - RequestContext.get().setSkipAuthorizationCheck(false); - } - - } +// public void createNewVersion() throws AtlasBaseException { +// AtlasVertex vertex = context.getVertex(entity.getGuid()); +// AtlasEntity existingContractEntity = entityRetriever.toAtlasEntity(vertex); +//// this.newEntity = new AtlasEntity(existingEntity); +// this.existingEntity.setAttribute(QUALIFIED_NAME, null); +// +// try { +// RequestContext.get().setSkipAuthorizationCheck(true); +// EntityStream entityStream = new AtlasEntityStream(existingEntity); +// entityStore.createOrUpdate(entityStream, false); +// LOG.info("Created bootstrap policies for connection {}", existingEntity.getAttribute(QUALIFIED_NAME)); +// } finally { +// RequestContext.get().setSkipAuthorizationCheck(false); +// } +// +// } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java index 63da3e909fe..e6d68f3895b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java @@ -8,10 +8,15 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSetter; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.exception.AtlasBaseException; import org.apache.commons.lang.StringUtils; +import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; + @JsonIgnoreProperties(ignoreUnknown = true) public class DataContract { @@ -124,6 +129,31 @@ public static final class Column { } + public static DataContract deserialize(String contractString) throws AtlasBaseException { + + if (StringUtils.isEmpty(contractString)) { + throw new AtlasBaseException(BAD_REQUEST, "Missing attribute: contract."); + } + + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS); + DataContract contract; + try { + contract = objectMapper.readValue(contractString, DataContract.class); + } catch (Exception ex) { + ex.printStackTrace(); + throw new AtlasBaseException("Failed at this"); + } + return contract; + } + + public static String serialize(DataContract contract) throws JsonProcessingException { + + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS); + return objectMapper.writeValueAsString(contract); + } + } From 4a4668c6c95072f995bafadeec93079e27233284 Mon Sep 17 00:00:00 2001 From: Rajat Movaliya Date: Wed, 3 Apr 2024 13:28:11 +0530 Subject: [PATCH 021/316] update lineage level correctly --- .../atlas/discovery/EntityLineageService.java | 43 ++++++++++--------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index b64739657b7..4a6eb708217 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -283,6 +283,7 @@ private AtlasLineageOnDemandInfo getLineageInfoOnDemand(String guid, AtlasLineag LineageOnDemandConstraints lineageConstraintsByGuid = getAndValidateLineageConstraintsByGuid(guid, atlasLineageOnDemandContext); AtlasLineageOnDemandInfo.LineageDirection direction = lineageConstraintsByGuid.getDirection(); + int level = 0; int depth = lineageConstraintsByGuid.getDepth(); AtlasLineageOnDemandInfo ret = initializeLineageOnDemandInfo(guid); @@ -296,21 +297,22 @@ private AtlasLineageOnDemandInfo getLineageInfoOnDemand(String guid, AtlasLineag if (isDataSet) { AtlasVertex datasetVertex = AtlasGraphUtilsV2.findByGuid(this.graph, guid); if (direction == AtlasLineageOnDemandInfo.LineageDirection.INPUT || direction == AtlasLineageOnDemandInfo.LineageDirection.BOTH) - traverseEdgesOnDemand(datasetVertex, true, depth, new HashSet<>(), atlasLineageOnDemandContext, ret, guid, inputEntitiesTraversed); + traverseEdgesOnDemand(datasetVertex, true, depth, level, new HashSet<>(), atlasLineageOnDemandContext, ret, guid, inputEntitiesTraversed); if (direction == AtlasLineageOnDemandInfo.LineageDirection.OUTPUT || direction == AtlasLineageOnDemandInfo.LineageDirection.BOTH) - traverseEdgesOnDemand(datasetVertex, false, depth, new HashSet<>(), atlasLineageOnDemandContext, ret, guid, outputEntitiesTraversed); + traverseEdgesOnDemand(datasetVertex, false, depth, level, new HashSet<>(), atlasLineageOnDemandContext, ret, guid, outputEntitiesTraversed); AtlasEntityHeader baseEntityHeader = entityRetriever.toAtlasEntityHeader(datasetVertex, atlasLineageOnDemandContext.getAttributes()); + baseEntityHeader.setDepth(level); ret.getGuidEntityMap().put(guid, baseEntityHeader); } else { AtlasVertex processVertex = AtlasGraphUtilsV2.findByGuid(this.graph, guid); // make one hop to the next dataset vertices from process vertex and traverse with 'depth = depth - 1' if (direction == AtlasLineageOnDemandInfo.LineageDirection.INPUT || direction == AtlasLineageOnDemandInfo.LineageDirection.BOTH) { Iterator processEdges = processVertex.getEdges(AtlasEdgeDirection.OUT, PROCESS_INPUTS_EDGE).iterator(); - traverseEdgesOnDemand(processEdges, true, depth, atlasLineageOnDemandContext, ret, processVertex, guid, inputEntitiesTraversed); + traverseEdgesOnDemand(processEdges, true, depth, level, atlasLineageOnDemandContext, ret, processVertex, guid, inputEntitiesTraversed); } if (direction == AtlasLineageOnDemandInfo.LineageDirection.OUTPUT || direction == AtlasLineageOnDemandInfo.LineageDirection.BOTH) { Iterator processEdges = processVertex.getEdges(AtlasEdgeDirection.OUT, PROCESS_OUTPUTS_EDGE).iterator(); - traverseEdgesOnDemand(processEdges, false, depth, atlasLineageOnDemandContext, ret, processVertex, guid, outputEntitiesTraversed); + traverseEdgesOnDemand(processEdges, false, depth, level, atlasLineageOnDemandContext, ret, processVertex, guid, outputEntitiesTraversed); } } RequestContext.get().endMetricRecord(metricRecorder); @@ -318,8 +320,10 @@ private AtlasLineageOnDemandInfo getLineageInfoOnDemand(String guid, AtlasLineag } - private void traverseEdgesOnDemand(Iterator processEdges, boolean isInput, int depth, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, AtlasVertex processVertex, String baseGuid, AtomicInteger entitiesTraversed) throws AtlasBaseException { + private void traverseEdgesOnDemand(Iterator processEdges, boolean isInput, int depth, int level, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, AtlasVertex processVertex, String baseGuid, AtomicInteger entitiesTraversed) throws AtlasBaseException { AtlasLineageOnDemandInfo.LineageDirection direction = isInput ? AtlasLineageOnDemandInfo.LineageDirection.INPUT : AtlasLineageOnDemandInfo.LineageDirection.OUTPUT; + int nextLevel = isInput ? level - 1: level + 1; + while (processEdges.hasNext()) { AtlasEdge processEdge = processEdges.next(); AtlasVertex datasetVertex = processEdge.getInVertex(); @@ -336,7 +340,7 @@ private void traverseEdgesOnDemand(Iterator processEdges, boolean isI if (incrementAndCheckIfRelationsLimitReached(processEdge, isInputEdge, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, direction)) { break; } else { - addEdgeToResult(processEdge, ret, atlasLineageOnDemandContext, depth); + addEdgeToResult(processEdge, ret, atlasLineageOnDemandContext, nextLevel); } String inGuid = AtlasGraphUtilsV2.getIdFromVertex(datasetVertex); @@ -346,17 +350,17 @@ private void traverseEdgesOnDemand(Iterator processEdges, boolean isI ret.getRelationsOnDemand().put(inGuid, new LineageInfoOnDemand(inGuidLineageConstrains)); } - traverseEdgesOnDemand(datasetVertex, isInput, depth - 1, new HashSet<>(), atlasLineageOnDemandContext, ret, baseGuid, entitiesTraversed); + traverseEdgesOnDemand(datasetVertex, isInput, depth - 1, nextLevel, new HashSet<>(), atlasLineageOnDemandContext, ret, baseGuid, entitiesTraversed); } } - private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, int depth, Set visitedVertices, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, String baseGuid, AtomicInteger entitiesTraversed) throws AtlasBaseException { + private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, int depth, int level, Set visitedVertices, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, String baseGuid, AtomicInteger entitiesTraversed) throws AtlasBaseException { if (isEntityTraversalLimitReached(entitiesTraversed)) return; if (depth != 0) { // base condition of recursion for depth AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("traverseEdgesOnDemand"); AtlasLineageOnDemandInfo.LineageDirection direction = isInput ? AtlasLineageOnDemandInfo.LineageDirection.INPUT : AtlasLineageOnDemandInfo.LineageDirection.OUTPUT; - + int nextLevel = isInput ? level - 1: level + 1; // keep track of visited vertices to avoid circular loop visitedVertices.add(getId(datasetVertex)); @@ -385,7 +389,7 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i else continue; } else { - addEdgeToResult(incomingEdge, ret, atlasLineageOnDemandContext, depth); + addEdgeToResult(incomingEdge, ret, atlasLineageOnDemandContext, level); } AtlasPerfMetrics.MetricRecorder traverseEdgesOnDemandGetEdgesOut = RequestContext.get().startMetricRecord("traverseEdgesOnDemandGetEdgesOut"); @@ -413,13 +417,13 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i else continue; } else { - addEdgeToResult(outgoingEdge, ret, atlasLineageOnDemandContext, depth); + addEdgeToResult(outgoingEdge, ret, atlasLineageOnDemandContext, nextLevel); entitiesTraversed.incrementAndGet(); if (isEntityTraversalLimitReached(entitiesTraversed)) setEntityLimitReachedFlag(isInput, ret); } if (entityVertex != null && !visitedVertices.contains(getId(entityVertex))) { - traverseEdgesOnDemand(entityVertex, isInput, depth - 1, visitedVertices, atlasLineageOnDemandContext, ret, baseGuid, entitiesTraversed); // execute inner depth + traverseEdgesOnDemand(entityVertex, isInput, depth - 1, nextLevel, visitedVertices, atlasLineageOnDemandContext, ret, baseGuid, entitiesTraversed); // execute inner depth } } } @@ -874,9 +878,9 @@ private void addEdgeToResult(AtlasEdge edge, AtlasLineageInfo lineageInfo, } } - private void addEdgeToResult(AtlasEdge edge, AtlasLineageOnDemandInfo lineageInfo, AtlasLineageOnDemandContext atlasLineageOnDemandContext, int depth) throws AtlasBaseException { + private void addEdgeToResult(AtlasEdge edge, AtlasLineageOnDemandInfo lineageInfo, AtlasLineageOnDemandContext atlasLineageOnDemandContext, int level) throws AtlasBaseException { if (!lineageContainsVisitedEdgeV2(lineageInfo, edge)) { - processEdge(edge, lineageInfo, atlasLineageOnDemandContext, depth); + processEdge(edge, lineageInfo, atlasLineageOnDemandContext, level); } } @@ -1446,11 +1450,11 @@ private void processEdge(final AtlasEdge edge, final Map entities, final Set relations, final Set visitedEdges, final Set attributes, int depth) throws AtlasBaseException { + private void processEdge(final AtlasEdge edge, final Map entities, final Set relations, final Set visitedEdges, final Set attributes, int level) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processEdge"); AtlasVertex inVertex = edge.getInVertex(); AtlasVertex outVertex = edge.getOutVertex(); @@ -1459,15 +1463,14 @@ private void processEdge(final AtlasEdge edge, final Map Date: Wed, 3 Apr 2024 14:36:30 +0530 Subject: [PATCH 022/316] remove depth from process node --- .../org/apache/atlas/discovery/EntityLineageService.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index 4a6eb708217..3ec77439061 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -1458,6 +1458,7 @@ private void processEdge(final AtlasEdge edge, final Map Date: Wed, 3 Apr 2024 18:07:53 +0530 Subject: [PATCH 023/316] Revert "changes to make it work" This reverts commit 2cfd40b51b12a7628b2d829c1adfe29676b2693b. --- .../org/apache/atlas/notification/AbstractNotification.java | 2 +- pom.xml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/notification/src/main/java/org/apache/atlas/notification/AbstractNotification.java b/notification/src/main/java/org/apache/atlas/notification/AbstractNotification.java index bed35932a11..cca4cb81d89 100644 --- a/notification/src/main/java/org/apache/atlas/notification/AbstractNotification.java +++ b/notification/src/main/java/org/apache/atlas/notification/AbstractNotification.java @@ -96,7 +96,7 @@ public void send(NotificationType type, List messages, MessageSource sour createNotificationMessages(messages.get(index), strMessages, source); } -// sendInternal(type, strMessages); + sendInternal(type, strMessages); } @Override diff --git a/pom.xml b/pom.xml index 3abc39041b7..1cc9aa70dc9 100644 --- a/pom.xml +++ b/pom.xml @@ -717,7 +717,7 @@ 4.4.13 2.12.4 2.12.4 - 0.6.04 + 0.6.03 0.5.3 1 3.1.0 From 94d98db894fd146166eddbcbfc529bfb1dca3e13 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Thu, 4 Apr 2024 11:04:03 +0530 Subject: [PATCH 024/316] Add unknown attributes to DataContract and fix minor bugs --- .../store/graph/v2/AtlasEntityStoreV2.java | 5 +- .../contract/ContractPreProcessor.java | 112 +++++++++--------- .../preprocessor/contract/DataContract.java | 55 +++++++-- 3 files changed, 105 insertions(+), 67 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 02dd3965b87..2b2c6a7a3c9 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -695,8 +695,9 @@ public EntityMutationResponse purgeByIds(Set guids) throws AtlasBaseExce if (CollectionUtils.isEmpty(guids)) { throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "Guid(s) not specified"); } - - AtlasAuthorizationUtils.verifyAccess(new AtlasAdminAccessRequest(AtlasPrivilege.ADMIN_PURGE), "purge entity: guids=", guids); + if (!RequestContext.get().isSkipAuthorizationCheck()) { + AtlasAuthorizationUtils.verifyAccess(new AtlasAdminAccessRequest(AtlasPrivilege.ADMIN_PURGE), "purge entity: guids=", guids); + } Collection purgeCandidates = new ArrayList<>(); for (String guid : guids) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index aecca4ea9d8..adfe3e4e2a8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -5,10 +5,12 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.atlas.RequestContext; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; import org.apache.atlas.model.instance.AtlasStruct; +import org.apache.atlas.model.instance.EntityMutationResponse; import org.apache.atlas.model.instance.EntityMutations; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; @@ -30,6 +32,9 @@ public class ContractPreProcessor extends AbstractContractPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(ContractPreProcessor.class); public static final String ATTR_CONTRACT = "contract"; + public static final String ATTR_VERSION = "contractVersion"; + public static final String ATTR_ASSET_QUALIFIED_NAME = "contractAssetQualifiedName"; + public static final String ATTR_PARENT_GUID = "parentGuid"; public static final String CONTRACT_QUALIFIED_NAME_SUFFIX = "contract"; public static final String VERSION_PREFIX = "version"; public static final String CONTRACT_ATTR_STATUS = "status"; @@ -53,6 +58,7 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co processCreateContract(entity, context); break; case UPDATE: + // Updating an existing version of the contract processUpdateContract(entity, context); } @@ -69,38 +75,6 @@ private void processUpdateContract(AtlasEntity entity, EntityMutationContext con } // Add cases for update in status field and certificateStatus } - - private List getDiffAttributes(EntityMutationContext context, AtlasEntity entity, AtlasEntity latestExistingVersion) throws AtlasBaseException { - AtlasEntityComparator entityComparator = new AtlasEntityComparator(typeRegistry, entityRetriever, context.getGuidAssignments(), true, true); - AtlasEntityComparator.AtlasEntityDiffResult diffResult = entityComparator.getDiffResult(entity, latestExistingVersion, false); - List attributesSet = new ArrayList<>(); - - if (diffResult.hasDifference()) { - for (Map.Entry entry : diffResult.getDiffEntity().getAttributes().entrySet()) { - if (!entry.getKey().equals(QUALIFIED_NAME)) { - attributesSet.add(entry.getKey()); - } - } - } - return attributesSet; - } - - private boolean isEqualContract(String firstNode, String secondNode) throws AtlasBaseException { - ObjectMapper mapper = new ObjectMapper(); - try { - JsonNode actualObj1 = mapper.readTree(firstNode); - JsonNode actualObj2 = mapper.readTree(secondNode); - //Ignore status field change - ((ObjectNode) actualObj1).remove(CONTRACT_ATTR_STATUS); - ((ObjectNode) actualObj2).remove(CONTRACT_ATTR_STATUS); - - return actualObj1.equals(actualObj2); - } catch (JsonProcessingException e) { - throw new AtlasBaseException(JSON_ERROR, e.getMessage()); - } - - } - private void processCreateContract(AtlasEntity entity, EntityMutationContext context) throws AtlasBaseException { /* Low-level Design @@ -116,6 +90,8 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con */ String contractQName = (String) entity.getAttribute(QUALIFIED_NAME); + validateAttribute(!contractQName.endsWith(String.format("/%s", CONTRACT_QUALIFIED_NAME_SUFFIX)), "Invalid qualifiedName for the contract."); + String contractString = (String) entity.getAttribute(ATTR_CONTRACT); DataContract contract = DataContract.deserialize(contractString); DataContract.Dataset dataset = contract.dataset; @@ -124,10 +100,9 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con authorizeContractCreateOrUpdate(entity, associatedAsset); contractAttributeSync(entity, contract); - contractString = updateContract(contract, contractString); + contractString = DataContract.serialize(contract); entity.setAttribute(ATTR_CONTRACT, contractString); - validateAttribute(!contractQName.endsWith(String.format("/%s", CONTRACT_QUALIFIED_NAME_SUFFIX)), "Invalid qualifiedName for the contract."); ContractVersionUtils versionUtil = new ContractVersionUtils(entity, context, entityRetriever, typeRegistry, entityStore, graph); AtlasEntity latestExistingVersion = versionUtil.getLatestVersion(); @@ -138,10 +113,7 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con Integer latestVersionNumber = Integer.valueOf(qName.substring(qName.lastIndexOf("/V") + 2)); List attributes = getDiffAttributes(context, entity, latestExistingVersion); if (attributes.isEmpty()) { - context.getCreatedEntities().remove(entity); - Set guids = new HashSet<>(); - guids.add(entity.getGuid()); - entityStore.purgeByIds(guids); + removeCreatingVertex(context, entity); return; } @@ -161,6 +133,9 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con } else { // Create New version of entity entity.setAttribute(QUALIFIED_NAME, String.format("%s/%s/V%s", contractQName, VERSION_PREFIX, ++latestVersionNumber)); + entity.setAttribute(ATTR_VERSION, String.format("V%s", latestVersionNumber)); + entity.setAttribute(ATTR_ASSET_QUALIFIED_NAME, associatedAsset.getEntity().getAttribute(QUALIFIED_NAME)); + entity.setAttribute(ATTR_PARENT_GUID, latestExistingVersion.getGuid()); } @@ -170,10 +145,11 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con } else { // Create new contract entity.setAttribute(QUALIFIED_NAME, String.format("%s/%s/%s", contractQName, VERSION_PREFIX, "V1")); + entity.setAttribute(ATTR_VERSION, "V1"); + entity.setAttribute(ATTR_ASSET_QUALIFIED_NAME, associatedAsset.getEntity().getAttribute(QUALIFIED_NAME)); } - entity.setAttribute("assetQualifiedName", associatedAsset.getEntity().getAttribute(QUALIFIED_NAME)); if (contract.getStatus() == DataContract.STATUS.VERIFIED && entity.getAttribute(ATTR_CERTIFICATE_STATUS).equals(DataContract.STATUS.VERIFIED.name())) { datasetAttributeSync(associatedAsset.getEntity(), contract); @@ -181,30 +157,39 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con } - private String updateContract(DataContract contract, String contractString) throws AtlasBaseException { - ObjectMapper objectMapper = new ObjectMapper(); - try { - String contractObjString = DataContract.serialize(contract); - Map mapJson = objectMapper.readValue(contractString, new TypeReference>() { - }); - Map mapTargetJson = objectMapper.readValue(contractObjString, new TypeReference>() { - }); - - for (Map.Entry keyVal : mapTargetJson.entrySet()) { - mapJson.replace(keyVal.getKey(), keyVal.getValue()); + private List getDiffAttributes(EntityMutationContext context, AtlasEntity entity, AtlasEntity latestExistingVersion) throws AtlasBaseException { + AtlasEntityComparator entityComparator = new AtlasEntityComparator(typeRegistry, entityRetriever, context.getGuidAssignments(), true, true); + AtlasEntityComparator.AtlasEntityDiffResult diffResult = entityComparator.getDiffResult(entity, latestExistingVersion, false); + List attributesSet = new ArrayList<>(); + + if (diffResult.hasDifference()) { + for (Map.Entry entry : diffResult.getDiffEntity().getAttributes().entrySet()) { + if (!entry.getKey().equals(QUALIFIED_NAME)) { + attributesSet.add(entry.getKey()); + } } - return objectMapper.writeValueAsString(mapJson); - } catch (JsonProcessingException ex) { - throw new AtlasBaseException(JSON_ERROR, ex.getMessage()); + } + return attributesSet; + } + + private boolean isEqualContract(String firstNode, String secondNode) throws AtlasBaseException { + ObjectMapper mapper = new ObjectMapper(); + try { + JsonNode actualObj1 = mapper.readTree(firstNode); + JsonNode actualObj2 = mapper.readTree(secondNode); + //Ignore status field change + ((ObjectNode) actualObj1).remove(CONTRACT_ATTR_STATUS); + ((ObjectNode) actualObj2).remove(CONTRACT_ATTR_STATUS); + + return actualObj1.equals(actualObj2); + } catch (JsonProcessingException e) { + throw new AtlasBaseException(JSON_ERROR, e.getMessage()); } } private void updateExistingVersion(EntityMutationContext context, AtlasEntity entity, AtlasEntity latestExistingVersion) throws AtlasBaseException { - context.getCreatedEntities().remove(entity); - Set guids = new HashSet<>(); - guids.add(entity.getGuid()); - entityStore.purgeByIds(guids); + removeCreatingVertex(context, entity); entity.setAttribute(QUALIFIED_NAME, latestExistingVersion.getAttribute(QUALIFIED_NAME)); entity.setGuid(latestExistingVersion.getGuid()); @@ -216,6 +201,19 @@ private void updateExistingVersion(EntityMutationContext context, AtlasEntity en } + private void removeCreatingVertex(EntityMutationContext context, AtlasEntity entity) throws AtlasBaseException { + context.getCreatedEntities().remove(entity); + try { + RequestContext.get().setSkipAuthorizationCheck(true); + Set guids = new HashSet<>(); + guids.add(entity.getGuid()); + entityStore.purgeByIds(guids); + } finally { + RequestContext.get().setSkipAuthorizationCheck(false); + } + + } + private void contractAttributeSync(AtlasEntity entity, DataContract contract) { // Sync certificateStatus if (!Objects.equals(entity.getAttribute(ATTR_CERTIFICATE_STATUS), contract.getStatus().name())) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java index e6d68f3895b..a478466b9b3 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java @@ -1,13 +1,13 @@ package org.apache.atlas.repository.store.graph.v2.preprocessor.contract; import java.lang.String; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSetter; +import com.fasterxml.jackson.annotation.*; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; @@ -16,9 +16,11 @@ import org.apache.commons.lang.StringUtils; import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; +import static org.apache.atlas.AtlasErrorCode.JSON_ERROR; @JsonIgnoreProperties(ignoreUnknown = true) +@JsonPropertyOrder({"kind", "status", "template_version", "dataset", "columns"}) public class DataContract { @JsonProperty(required = true) public String kind; @@ -27,7 +29,7 @@ public class DataContract { public String templateVersion; public Dataset dataset; public List columns; - + private Map unknownFields = new HashMap<>(); public STATUS getStatus() { return status; } @@ -37,6 +39,16 @@ public void setStatus(STATUS status) { this.status = status; } + @JsonAnySetter + public void setUnknownFields(String key, Object value) { + unknownFields.put(key, value); + } + + @JsonAnyGetter + public Map getUnknownFields() { + return unknownFields; + } + public enum STATUS { @JsonProperty("DRAFT") DRAFT, @JsonProperty("VERIFIED") VERIFIED; @@ -81,12 +93,23 @@ private boolean isSemVer(String version) { } @JsonIgnoreProperties(ignoreUnknown = true) + @JsonPropertyOrder({"name", "type", "description"}) public static final class Dataset { public String name; @JsonProperty(required = true) public DATASET_TYPE type; public String description; + private Map unknownFields = new HashMap<>(); + + @JsonAnySetter + public void setUnknownFields(String key, Object value) { + unknownFields.put(key, value); + } + @JsonAnyGetter + public Map getUnknownFields() { + return unknownFields; + } @JsonSetter("type") public void setType(DATASET_TYPE type) { @@ -117,6 +140,7 @@ public static DATASET_TYPE from(String s) throws AtlasBaseException { } @JsonIgnoreProperties(ignoreUnknown = true) + @JsonPropertyOrder({"name", "description", "data_type"}) public static final class Column { public String name; @@ -125,6 +149,17 @@ public static final class Column { public boolean is_primary; public String data_type; + private Map unknownFields = new HashMap<>(); + + @JsonAnySetter + public void setUnknownFields(String key, Object value) { + unknownFields.put(key, value); + } + @JsonAnyGetter + public Map getUnknownFields() { + return unknownFields; + } + } @@ -147,11 +182,15 @@ public static DataContract deserialize(String contractString) throws AtlasBaseEx return contract; } - public static String serialize(DataContract contract) throws JsonProcessingException { + public static String serialize(DataContract contract) throws AtlasBaseException { - ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS); - return objectMapper.writeValueAsString(contract); + try { + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS); + return objectMapper.writeValueAsString(contract); + } catch (JsonProcessingException ex) { + throw new AtlasBaseException(JSON_ERROR, ex.getMessage()); + } } From d64e9690e9351fc0e6c5670262a2a1b27ee82da7 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Thu, 4 Apr 2024 12:10:16 +0530 Subject: [PATCH 025/316] Add attribute hasContract for assets --- .../contract/ContractPreProcessor.java | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index adfe3e4e2a8..38c8140008c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -35,6 +35,7 @@ public class ContractPreProcessor extends AbstractContractPreProcessor { public static final String ATTR_VERSION = "contractVersion"; public static final String ATTR_ASSET_QUALIFIED_NAME = "contractAssetQualifiedName"; public static final String ATTR_PARENT_GUID = "parentGuid"; + public static final String ATTR_HAS_CONTRACT = "hasContract"; public static final String CONTRACT_QUALIFIED_NAME_SUFFIX = "contract"; public static final String VERSION_PREFIX = "version"; public static final String CONTRACT_ATTR_STATUS = "status"; @@ -123,7 +124,7 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con updateExistingVersion(context, entity, latestExistingVersion); } // Contract status changed, either to draft or verified - return; + } else if (attributes.contains(ATTR_CONTRACT)) { //Contract is changed if (isEqualContract(contractString, (String) latestExistingVersion.getAttribute(ATTR_CONTRACT))) { @@ -138,8 +139,6 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con entity.setAttribute(ATTR_PARENT_GUID, latestExistingVersion.getGuid()); } - - return; } } else { @@ -149,11 +148,7 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con entity.setAttribute(ATTR_ASSET_QUALIFIED_NAME, associatedAsset.getEntity().getAttribute(QUALIFIED_NAME)); } - - if (contract.getStatus() == DataContract.STATUS.VERIFIED && - entity.getAttribute(ATTR_CERTIFICATE_STATUS).equals(DataContract.STATUS.VERIFIED.name())) { - datasetAttributeSync(associatedAsset.getEntity(), contract); - } + datasetAttributeSync(associatedAsset.getEntity(), contract, entity); } @@ -240,11 +235,21 @@ private void contractAttributeSync(AtlasEntity entity, DataContract contract) { } - private void datasetAttributeSync(AtlasEntity associatedAsset, DataContract contract) throws AtlasBaseException { - - DataContract.Dataset dataset = contract.dataset; - // Will implement dataset attribute sync from the contract attributes - + private void datasetAttributeSync(AtlasEntity associatedAsset, DataContract contract, AtlasEntity contractAsset) throws AtlasBaseException { + associatedAsset.setAttribute(ATTR_HAS_CONTRACT, true); + if (contract.getStatus() == DataContract.STATUS.VERIFIED && + contractAsset.getAttribute(ATTR_CERTIFICATE_STATUS).equals(DataContract.STATUS.VERIFIED.name())) { + DataContract.Dataset dataset = contract.dataset; + // Will implement dataset attribute sync from the contract attributes + } + try { + RequestContext.get().setSkipAuthorizationCheck(true); + EntityStream entityStream = new AtlasEntityStream(associatedAsset); + entityStore.createOrUpdate(entityStream, false); + LOG.info("Updated associated asset attributes of contract {}", associatedAsset.getAttribute(QUALIFIED_NAME)); + } finally { + RequestContext.get().setSkipAuthorizationCheck(false); + } } private static void validateAttribute(boolean isInvalid, String errorMessage) throws AtlasBaseException { From d3ff72d145e9a5ed475203dab8824eaa6510b273 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Mon, 8 Apr 2024 22:35:02 +0530 Subject: [PATCH 026/316] pre-processor for domain create --- .../datamesh/AbstractDomainPreProcessor.java | 23 +++++++++++++ .../datamesh/DomainPreProcessor.java | 33 ++++++++++++++++--- 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 546d6e63bcd..ed529ef5546 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -42,6 +42,8 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import static org.apache.atlas.repository.Constants.NAME; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; @@ -122,6 +124,27 @@ protected void isAuthorized(AtlasEntityHeader sourceDomain, AtlasEntityHeader ta "delete on target Domain: ", targetDomain.getAttribute(NAME)); } + public static String toCamelCase(String domainName) { + domainName = domainName.toLowerCase(); + Pattern pattern = Pattern.compile("[^a-zA-Z0-9]+(.*)"); + Matcher matcher = pattern.matcher(domainName); + + StringBuilder camelCaseString = new StringBuilder(); + + int lastIndex = 0; + while (matcher.find()) { + camelCaseString.append(domainName, lastIndex, matcher.start()); + camelCaseString.append(Character.toUpperCase(matcher.group(1).charAt(0))); + + lastIndex = matcher.end(); + } + + // Append the remaining substring + camelCaseString.append(domainName.substring(lastIndex)); + + return camelCaseString.toString(); + } + /** * Record the updated child entities, it will be used to send notification and store audit logs * @param entityVertex Child entity vertex diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index c5b26ad8ce8..8e1d68b9d03 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -76,13 +76,38 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co setParent(entity, context); - if (operation == EntityMutations.EntityOperation.UPDATE) { - processUpdateDomain(entity, vertex); - } else { - LOG.error("DataProductPreProcessor.processAttributes: Operation not supported {}", operation); + switch (operation) { + case CREATE: + processCreateDomain(entity, vertex); + break; + case UPDATE: + processUpdateDomain(entity, vertex); + break; } } + private void processCreateDomain(AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreateDomain"); + String domainName = (String) entity.getAttribute(NAME); + String parentDomainQualifiedName = (String) entity.getAttribute(PARENT_DOMAIN_QN); + + domainExists(domainName, parentDomainQualifiedName); + entity.setAttribute(QUALIFIED_NAME, createQualifiedName(parentDomainQualifiedName, domainName)); + + RequestContext.get().endMetricRecord(metricRecorder); + } + + public static String createQualifiedName(String parentDomainQualifiedName, String domainName) { + if (StringUtils.isNotEmpty(parentDomainQualifiedName)) { + return parentDomainQualifiedName + "/" + getUUID(); + } else{ + String camelDomainName = toCamelCase(domainName); + String prefixQN = "default/domain"+"/"+camelDomainName; + return prefixQN + "/" + getUUID(); + } + } + + private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdateDomain"); String domainName = (String) entity.getAttribute(NAME); From 6ee00aa7edf32c73e2c778b120589f0ab108ca24 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Tue, 9 Apr 2024 11:53:37 +0530 Subject: [PATCH 027/316] Add validation of data contract spec --- repository/pom.xml | 6 ++ .../contract/ContractPreProcessor.java | 36 +++++---- .../preprocessor/contract/DataContract.java | 79 +++++++++++++------ 3 files changed, 80 insertions(+), 41 deletions(-) diff --git a/repository/pom.xml b/repository/pom.xml index 10d8d876fb1..64cb9c0d48c 100755 --- a/repository/pom.xml +++ b/repository/pom.xml @@ -322,6 +322,12 @@ 3.0.0-SNAPSHOT + + org.hibernate + hibernate-validator + 4.3.0.Final + + diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index 38c8140008c..fd82d9edd04 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -1,7 +1,6 @@ package org.apache.atlas.repository.store.graph.v2.preprocessor.contract; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -10,7 +9,6 @@ import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; import org.apache.atlas.model.instance.AtlasStruct; -import org.apache.atlas.model.instance.EntityMutationResponse; import org.apache.atlas.model.instance.EntityMutations; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; @@ -35,7 +33,9 @@ public class ContractPreProcessor extends AbstractContractPreProcessor { public static final String ATTR_VERSION = "contractVersion"; public static final String ATTR_ASSET_QUALIFIED_NAME = "contractAssetQualifiedName"; public static final String ATTR_PARENT_GUID = "parentGuid"; - public static final String ATTR_HAS_CONTRACT = "hasContract"; + public static final String ASSET_ATTR_HAS_CONTRACT = "hasContract"; + public static final String ASSET_ATTR_DESCRIPTION = "description"; + public static final String CONTRACT_QUALIFIED_NAME_SUFFIX = "contract"; public static final String VERSION_PREFIX = "version"; public static final String CONTRACT_ATTR_STATUS = "status"; @@ -69,7 +69,7 @@ private void processUpdateContract(AtlasEntity entity, EntityMutationContext con String contractString = (String) entity.getAttribute(ATTR_CONTRACT); AtlasVertex vertex = context.getVertex(entity.getGuid()); AtlasEntity existingContractEntity = entityRetriever.toAtlasEntity(vertex); - + // TODO: Check for qualifiedName to understand if a particular version is getting updated or duplicate contract in payload if (!isEqualContract(contractString, (String) existingContractEntity.getAttribute(ATTR_CONTRACT))) { // Update the same asset(entity) throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Can't update a specific version of contract"); @@ -148,7 +148,7 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con entity.setAttribute(ATTR_ASSET_QUALIFIED_NAME, associatedAsset.getEntity().getAttribute(QUALIFIED_NAME)); } - datasetAttributeSync(associatedAsset.getEntity(), contract, entity); + datasetAttributeSync(context, associatedAsset.getEntity(), contract, entity); } @@ -209,7 +209,7 @@ private void removeCreatingVertex(EntityMutationContext context, AtlasEntity ent } - private void contractAttributeSync(AtlasEntity entity, DataContract contract) { + private void contractAttributeSync(AtlasEntity entity, DataContract contract) throws AtlasBaseException { // Sync certificateStatus if (!Objects.equals(entity.getAttribute(ATTR_CERTIFICATE_STATUS), contract.getStatus().name())) { /* @@ -223,32 +223,38 @@ private void contractAttributeSync(AtlasEntity entity, DataContract contract) { */ if (Objects.equals(entity.getAttribute(ATTR_CERTIFICATE_STATUS), DataContract.STATUS.VERIFIED.name())) { - contract.setStatus(DataContract.STATUS.VERIFIED); + contract.setStatus(String.valueOf(DataContract.STATUS.VERIFIED)); } else if (Objects.equals(contract.getStatus(), DataContract.STATUS.VERIFIED)) { entity.setAttribute(ATTR_CERTIFICATE_STATUS, DataContract.STATUS.VERIFIED.name()); } else { entity.setAttribute(ATTR_CERTIFICATE_STATUS, DataContract.STATUS.DRAFT); - contract.setStatus(DataContract.STATUS.DRAFT); + contract.setStatus(String.valueOf(DataContract.STATUS.DRAFT)); } } } - private void datasetAttributeSync(AtlasEntity associatedAsset, DataContract contract, AtlasEntity contractAsset) throws AtlasBaseException { - associatedAsset.setAttribute(ATTR_HAS_CONTRACT, true); + private void datasetAttributeSync(EntityMutationContext context, AtlasEntity associatedAsset, DataContract contract, AtlasEntity contractAsset) throws AtlasBaseException { + associatedAsset.setAttribute(ASSET_ATTR_HAS_CONTRACT, true); if (contract.getStatus() == DataContract.STATUS.VERIFIED && contractAsset.getAttribute(ATTR_CERTIFICATE_STATUS).equals(DataContract.STATUS.VERIFIED.name())) { DataContract.Dataset dataset = contract.dataset; // Will implement dataset attribute sync from the contract attributes + if (!dataset.description.isEmpty()) { + associatedAsset.setAttribute(ASSET_ATTR_DESCRIPTION, dataset.description); + } } try { - RequestContext.get().setSkipAuthorizationCheck(true); - EntityStream entityStream = new AtlasEntityStream(associatedAsset); - entityStore.createOrUpdate(entityStream, false); - LOG.info("Updated associated asset attributes of contract {}", associatedAsset.getAttribute(QUALIFIED_NAME)); + AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(associatedAsset.getGuid()); + AtlasEntityType entityType = ensureEntityType(associatedAsset.getTypeName()); + context.addUpdated(associatedAsset.getGuid(), associatedAsset, entityType, vertex); +// RequestContext.get().setSkipAuthorizationCheck(true); +// EntityStream entityStream = new AtlasEntityStream(associatedAsset); +// entityStore.createOrUpdate(entityStream, false); +// LOG.info("Updated associated asset attributes of contract {}", associatedAsset.getAttribute(QUALIFIED_NAME)); } finally { - RequestContext.get().setSkipAuthorizationCheck(false); +// RequestContext.get().setSkipAuthorizationCheck(false); } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java index a478466b9b3..4e0887320c5 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java @@ -1,9 +1,7 @@ package org.apache.atlas.repository.store.graph.v2.preprocessor.contract; import java.lang.String; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -15,28 +13,39 @@ import org.apache.atlas.exception.AtlasBaseException; import org.apache.commons.lang.StringUtils; -import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; -import static org.apache.atlas.AtlasErrorCode.JSON_ERROR; +import javax.validation.*; +import javax.validation.constraints.NotNull; +import java.util.Set; + +import static org.apache.atlas.AtlasErrorCode.*; @JsonIgnoreProperties(ignoreUnknown = true) @JsonPropertyOrder({"kind", "status", "template_version", "dataset", "columns"}) public class DataContract { - @JsonProperty(required = true) - public String kind; - public STATUS status; + @Valid @NotNull + public String kind; + + public STATUS status; + @JsonProperty(value = "template_version", defaultValue = "0.0.1") - public String templateVersion; - public Dataset dataset; - public List columns; - private Map unknownFields = new HashMap<>(); + public String templateVersion; + @Valid @NotNull + public Dataset dataset; + @Valid + public List columns; + private Map unknownFields = new HashMap<>(); public STATUS getStatus() { return status; } @JsonSetter("status") - public void setStatus(STATUS status) { - this.status = status; + public void setStatus(String status) throws AtlasBaseException { + try { + this.status = STATUS.from(status); + } catch (IllegalArgumentException ex) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "status " + status + " is inappropriate. Accepted values: " + Arrays.toString(STATUS.values())); + } } @JsonAnySetter @@ -57,7 +66,6 @@ public static STATUS from(String s) { if(StringUtils.isEmpty(s)) { return DRAFT; } - switch (s.toLowerCase()) { case "draft": return DRAFT; @@ -79,9 +87,9 @@ public void setKind(String kind) throws AtlasBaseException { this.kind = kind; } - public void setTemplateVersion(String templateVersion) { + public void setTemplateVersion(String templateVersion) throws AtlasBaseException { if (!isSemVer(templateVersion)) { - throw new IllegalArgumentException("Invalid version syntax"); + throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "Invalid template_version syntax"); } this.templateVersion = templateVersion; } @@ -95,8 +103,9 @@ private boolean isSemVer(String version) { @JsonIgnoreProperties(ignoreUnknown = true) @JsonPropertyOrder({"name", "type", "description"}) public static final class Dataset { + @NotNull public String name; - @JsonProperty(required = true) + @NotNull public DATASET_TYPE type; public String description; private Map unknownFields = new HashMap<>(); @@ -112,8 +121,12 @@ public Map getUnknownFields() { } @JsonSetter("type") - public void setType(DATASET_TYPE type) { - this.type = type; + public void setType(String type) throws AtlasBaseException { + try { + this.type = DATASET_TYPE.from(type); + } catch (IllegalArgumentException | AtlasBaseException ex) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "dataset.type " + type + " is inappropriate. Accepted values: " + Arrays.toString(DATASET_TYPE.values())); + } } public enum DATASET_TYPE { @@ -131,7 +144,7 @@ public static DATASET_TYPE from(String s) throws AtlasBaseException { case "materialisedview": return MaterialisedView; default: - throw new AtlasBaseException("dataset.type value not supported yet."); + throw new AtlasBaseException(String.format("dataset.type: %s value not supported yet.", s)); } } } @@ -142,6 +155,7 @@ public static DATASET_TYPE from(String s) throws AtlasBaseException { @JsonIgnoreProperties(ignoreUnknown = true) @JsonPropertyOrder({"name", "description", "data_type"}) public static final class Column { + @NotNull public String name; public String description; @@ -160,8 +174,6 @@ public Map getUnknownFields() { return unknownFields; } - - } public static DataContract deserialize(String contractString) throws AtlasBaseException { @@ -175,11 +187,26 @@ public static DataContract deserialize(String contractString) throws AtlasBaseEx DataContract contract; try { contract = objectMapper.readValue(contractString, DataContract.class); - } catch (Exception ex) { - ex.printStackTrace(); - throw new AtlasBaseException("Failed at this"); + } catch (JsonProcessingException ex) { + throw new AtlasBaseException(ex.getOriginalMessage()); } + contract.validate(); return contract; + + } + + public void validate() throws AtlasBaseException { + Validator validator = Validation.buildDefaultValidatorFactory().getValidator(); + Set> violations = validator.validate(this); + if (!violations.isEmpty()) { + List errorMessageList = new ArrayList<>(); + for (ConstraintViolation violation : violations) { + errorMessageList.add(String.format("Field: %s -> %s", violation.getPropertyPath(), violation.getMessage())); + System.out.println(violation.getMessage()); + } + throw new AtlasBaseException(StringUtils.join(errorMessageList, "; ")); + } + } public static String serialize(DataContract contract) throws AtlasBaseException { From 67a189523c7105b3bcdafe202252b9b796df9686 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Tue, 9 Apr 2024 15:56:39 +0530 Subject: [PATCH 028/316] feat: include classification names option --- .../apache/atlas/model/discovery/SearchParams.java | 10 ++++++++++ .../apache/atlas/repository/graph/GraphHelper.java | 13 ++++++++++++- .../store/graph/v2/EntityGraphRetriever.java | 9 +++++++-- .../main/java/org/apache/atlas/RequestContext.java | 10 ++++++++++ .../org/apache/atlas/web/rest/DiscoveryREST.java | 1 + 5 files changed, 40 insertions(+), 3 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/model/discovery/SearchParams.java b/intg/src/main/java/org/apache/atlas/model/discovery/SearchParams.java index 57000cc84f0..93e6ea6b88b 100644 --- a/intg/src/main/java/org/apache/atlas/model/discovery/SearchParams.java +++ b/intg/src/main/java/org/apache/atlas/model/discovery/SearchParams.java @@ -19,6 +19,8 @@ public class SearchParams { boolean excludeMeanings; boolean excludeClassifications; + boolean includeClassificationNames = false; + RequestMetadata requestMetadata = new RequestMetadata(); Async async = new Async(); @@ -100,6 +102,14 @@ public void setExcludeMeanings(boolean excludeMeanings) { this.excludeMeanings = excludeMeanings; } + public boolean isIncludeClassificationNames() { + return includeClassificationNames; + } + + public void setIncludeClassificationNames(boolean includeClassificationNames) { + this.includeClassificationNames = includeClassificationNames; + } + public boolean isSaveSearchLog() { return requestMetadata.saveSearchLog; } diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index 987ceec296c..8844d2d87bc 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -785,7 +785,18 @@ public static List getTraitNames(AtlasVertex entityVertex) { public static List getPropagatedTraitNames(AtlasVertex entityVertex) { return getTraitNames(entityVertex, true); } - + public static List getAllTraitNamesFromAttribute(AtlasVertex entityVertex) { + List ret = new ArrayList<>(); + List traitNames = entityVertex.getMultiValuedProperty(TRAIT_NAMES_PROPERTY_KEY, String.class); + if (traitNames != null) { + ret.addAll(traitNames); + } + List propagatedTraitNames = entityVertex.getMultiValuedProperty(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, String.class); + if (propagatedTraitNames != null) { + ret.addAll(propagatedTraitNames); + } + return ret; + } public static List getAllTraitNames(AtlasVertex entityVertex) { return getTraitNames(entityVertex, null); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 691b5ab75f3..43b2aa80750 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1019,8 +1019,13 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, ret.setTypeName(typeName); ret.setGuid(guid); ret.setStatus(GraphHelper.getStatus(entityVertex)); - if(RequestContext.get().includeClassifications()){ - ret.setClassificationNames(getAllTraitNames(entityVertex)); + RequestContext context = RequestContext.get(); + boolean includeClassifications = context.includeClassifications(); + boolean includeClassificationNames = context.isIncludeClassificationNames(); + if(includeClassifications){ + ret.setClassificationNames(getAllTraitNamesFromAttribute(entityVertex)); + } else if (!includeClassifications && includeClassificationNames) { + ret.setClassifications(getAllClassifications(entityVertex)); } ret.setIsIncomplete(isIncomplete); ret.setLabels(getLabels(entityVertex)); diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java b/server-api/src/main/java/org/apache/atlas/RequestContext.java index 1c7ccababb6..d27be167598 100644 --- a/server-api/src/main/java/org/apache/atlas/RequestContext.java +++ b/server-api/src/main/java/org/apache/atlas/RequestContext.java @@ -88,6 +88,8 @@ public class RequestContext { private boolean allowDeletedRelationsIndexsearch = false; private boolean includeMeanings = true; private boolean includeClassifications = true; + + private boolean includeClassificationNames = false; private String currentTypePatchAction = ""; private AtlasTask currentTask; private String traceId; @@ -719,6 +721,14 @@ public boolean isCacheEnabled() { return this.cacheEnabled; } + public boolean isIncludeClassificationNames() { + return includeClassificationNames; + } + + public void setIncludeClassificationNames(boolean includeClassificationNames) { + this.includeClassificationNames = includeClassificationNames; + } + public class EntityGuidPair { private final Object entity; private final String guid; diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java index 7717baa49ac..70c1f57cc3a 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java @@ -393,6 +393,7 @@ public AtlasSearchResult indexSearch(@Context HttpServletRequest servletRequest, RequestContext.get().setIncludeMeanings(!parameters.isExcludeMeanings()); RequestContext.get().setIncludeClassifications(!parameters.isExcludeClassifications()); + RequestContext.get().setIncludeClassificationNames(parameters.isIncludeClassificationNames()); try { if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "DiscoveryREST.indexSearch(" + parameters + ")"); From 8201a16f18b4768d796a39dac9332b8aac166d05 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Tue, 9 Apr 2024 16:42:50 +0530 Subject: [PATCH 029/316] removed asset name from QN --- pom.xml | 2 +- .../datamesh/AbstractDomainPreProcessor.java | 21 ---------- .../datamesh/DataProductPreProcessor.java | 39 ++++++++++++++++--- .../datamesh/DomainPreProcessor.java | 14 +++---- 4 files changed, 41 insertions(+), 35 deletions(-) diff --git a/pom.xml b/pom.xml index 1cc9aa70dc9..3abc39041b7 100644 --- a/pom.xml +++ b/pom.xml @@ -717,7 +717,7 @@ 4.4.13 2.12.4 2.12.4 - 0.6.03 + 0.6.04 0.5.3 1 3.1.0 diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index ed529ef5546..f911b0c4a19 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -124,27 +124,6 @@ protected void isAuthorized(AtlasEntityHeader sourceDomain, AtlasEntityHeader ta "delete on target Domain: ", targetDomain.getAttribute(NAME)); } - public static String toCamelCase(String domainName) { - domainName = domainName.toLowerCase(); - Pattern pattern = Pattern.compile("[^a-zA-Z0-9]+(.*)"); - Matcher matcher = pattern.matcher(domainName); - - StringBuilder camelCaseString = new StringBuilder(); - - int lastIndex = 0; - while (matcher.find()) { - camelCaseString.append(domainName, lastIndex, matcher.start()); - camelCaseString.append(Character.toUpperCase(matcher.group(1).charAt(0))); - - lastIndex = matcher.end(); - } - - // Append the remaining substring - camelCaseString.append(domainName.substring(lastIndex)); - - return camelCaseString.toString(); - } - /** * Record the updated child entities, it will be used to send notification and store audit logs * @param entityVertex Child entity vertex diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index c806d1f179f..1018afb2c36 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -36,7 +36,7 @@ public DataProductPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetrie public void processAttributes(AtlasStruct entityStruct, EntityMutationContext context, EntityMutations.EntityOperation operation) throws AtlasBaseException { //Handle name & qualifiedName - if (operation == EntityMutations.EntityOperation.UPDATE && LOG.isDebugEnabled()) { + if (LOG.isDebugEnabled()) { LOG.debug("DataProductPreProcessor.processAttributes: pre processing {}, {}", entityStruct.getAttribute(QUALIFIED_NAME), operation); } @@ -47,10 +47,39 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co setParent(entity, context); - if (operation == EntityMutations.EntityOperation.UPDATE) { - processUpdateDomain(entity, vertex); - } else { - LOG.error("DataProductPreProcessor.processAttributes: Operation not supported {}", operation); + switch (operation) { + case CREATE: + processCreateProduct(entity, vertex); + break; + case UPDATE: + processUpdateDomain(entity, vertex); + break; + } + } + + private void processCreateProduct(AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreateDomain"); + String domainName = (String) entity.getAttribute(NAME); + String parentDomainQualifiedName = (String) entity.getAttribute(PARENT_DOMAIN_QN); + + productExists(domainName, parentDomainQualifiedName); + String newQualifiedName = createQualifiedName(parentDomainQualifiedName); + if(!newQualifiedName.isEmpty()){ + entity.setAttribute(QUALIFIED_NAME, newQualifiedName); + } + else{ + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Parent Domain Qualified Name is empty"); + } + + RequestContext.get().endMetricRecord(metricRecorder); + } + + public static String createQualifiedName(String parentDomainQualifiedName) { + if (StringUtils.isNotEmpty(parentDomainQualifiedName) && parentDomainQualifiedName !=null) { + return parentDomainQualifiedName + "/product/" + getUUID(); + } + else{ + return ""; } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index 8e1d68b9d03..0c2b98aa36a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -64,7 +64,7 @@ public DomainPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever e public void processAttributes(AtlasStruct entityStruct, EntityMutationContext context, EntityMutations.EntityOperation operation) throws AtlasBaseException { //Handle name & qualifiedName - if (operation == EntityMutations.EntityOperation.UPDATE && LOG.isDebugEnabled()) { + if (LOG.isDebugEnabled()) { LOG.debug("DomainPreProcessor.processAttributes: pre processing {}, {}", entityStruct.getAttribute(QUALIFIED_NAME), operation); } @@ -92,22 +92,20 @@ private void processCreateDomain(AtlasEntity entity, AtlasVertex vertex) throws String parentDomainQualifiedName = (String) entity.getAttribute(PARENT_DOMAIN_QN); domainExists(domainName, parentDomainQualifiedName); - entity.setAttribute(QUALIFIED_NAME, createQualifiedName(parentDomainQualifiedName, domainName)); + entity.setAttribute(QUALIFIED_NAME, createQualifiedName(parentDomainQualifiedName)); RequestContext.get().endMetricRecord(metricRecorder); } - public static String createQualifiedName(String parentDomainQualifiedName, String domainName) { - if (StringUtils.isNotEmpty(parentDomainQualifiedName)) { - return parentDomainQualifiedName + "/" + getUUID(); + public static String createQualifiedName(String parentDomainQualifiedName) { + if (StringUtils.isNotEmpty(parentDomainQualifiedName) && parentDomainQualifiedName !=null) { + return parentDomainQualifiedName + "/domain/" + getUUID(); } else{ - String camelDomainName = toCamelCase(domainName); - String prefixQN = "default/domain"+"/"+camelDomainName; + String prefixQN = "default/domain"; return prefixQN + "/" + getUUID(); } } - private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdateDomain"); String domainName = (String) entity.getAttribute(NAME); From 541efc1ab4072ec1c79d6139382c54916a361ad3 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Tue, 9 Apr 2024 16:45:17 +0530 Subject: [PATCH 030/316] removed pom.xml change --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3abc39041b7..1cc9aa70dc9 100644 --- a/pom.xml +++ b/pom.xml @@ -717,7 +717,7 @@ 4.4.13 2.12.4 2.12.4 - 0.6.04 + 0.6.03 0.5.3 1 3.1.0 From 7e5456007e7a6c49fdcc15b309eab5424990da72 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Tue, 9 Apr 2024 16:46:56 +0530 Subject: [PATCH 031/316] removed unused imports --- .../v2/preprocessor/datamesh/AbstractDomainPreProcessor.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index f911b0c4a19..546d6e63bcd 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -42,8 +42,6 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; -import java.util.regex.Matcher; -import java.util.regex.Pattern; import static org.apache.atlas.repository.Constants.NAME; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; From bc9269ee44b76bd0651abb46fd61f1967b89f00f Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Tue, 9 Apr 2024 18:43:28 +0530 Subject: [PATCH 032/316] fix: add classificationNames not classifcations --- .../atlas/repository/store/graph/v2/EntityGraphRetriever.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 43b2aa80750..90e041b4737 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1025,7 +1025,7 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, if(includeClassifications){ ret.setClassificationNames(getAllTraitNamesFromAttribute(entityVertex)); } else if (!includeClassifications && includeClassificationNames) { - ret.setClassifications(getAllClassifications(entityVertex)); + ret.setClassificationNames(getAllTraitNamesFromAttribute(entityVertex)); } ret.setIsIncomplete(isIncomplete); ret.setLabels(getLabels(entityVertex)); From 1c6216af413218838ce0d393847b44df2e48161e Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Tue, 16 Apr 2024 14:33:42 +0530 Subject: [PATCH 033/316] added updatePolicy logic --- .../datamesh/DataProductPreProcessor.java | 75 ++++++++++++++-- .../datamesh/DomainPreProcessor.java | 88 ++++++++++++++++--- 2 files changed, 143 insertions(+), 20 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 1018afb2c36..057de37acac 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -9,6 +9,7 @@ import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; +import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; @@ -21,6 +22,7 @@ import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.*; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_RESOURCES; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; public class DataProductPreProcessor extends AbstractDomainPreProcessor { @@ -52,7 +54,7 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co processCreateProduct(entity, vertex); break; case UPDATE: - processUpdateDomain(entity, vertex); + processUpdateDomain(entity, vertex, context); break; } } @@ -83,7 +85,7 @@ public static String createQualifiedName(String parentDomainQualifiedName) { } } - private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { + private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdateDomain"); String productName = (String) entity.getAttribute(NAME); String vertexQnName = vertex.getProperty(QUALIFIED_NAME, String.class); @@ -114,7 +116,7 @@ private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws //Auth check isAuthorized(currentParentDomainHeader, parentDomain); - processMoveDataProductToAnotherDomain(entity, currentParentDomainQualifiedName, newParentDomainQualifiedName, vertexQnName, superDomainQualifiedName); + processMoveDataProductToAnotherDomain(entity, currentParentDomainQualifiedName, newParentDomainQualifiedName, vertexQnName, superDomainQualifiedName, context); entity.setAttribute(PARENT_DOMAIN_QN, newParentDomainQualifiedName); } else { @@ -129,10 +131,11 @@ private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws } private void processMoveDataProductToAnotherDomain(AtlasEntity product, - String sourceDomainQualifiedName, - String targetDomainQualifiedName, - String currentDataProductQualifiedName, - String superDomainQualifiedName) throws AtlasBaseException { + String sourceDomainQualifiedName, + String targetDomainQualifiedName, + String currentDataProductQualifiedName, + String superDomainQualifiedName, + EntityMutationContext context) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("processMoveDataProductToAnotherDomain"); try { @@ -148,6 +151,9 @@ private void processMoveDataProductToAnotherDomain(AtlasEntity product, product.setAttribute(PARENT_DOMAIN_QN, targetDomainQualifiedName); product.setAttribute(SUPER_DOMAIN_QN, superDomainQualifiedName); + //Update policy + updatePolicy(currentDataProductQualifiedName, updatedQualifiedName, context); + LOG.info("Moved dataProduct {} to Domain {}", domainName, targetDomainQualifiedName); } finally { @@ -155,6 +161,59 @@ private void processMoveDataProductToAnotherDomain(AtlasEntity product, } } + private void updatePolicy(String currentQualifiedName, String updatedQualifiedName, EntityMutationContext context) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("updateDomainPolicy"); + try { + LOG.info("Updating policy for entity {}", currentQualifiedName); + Map updatedpolicyResources = new HashMap<>(); + + String currentResource = "entity:"+ currentQualifiedName; + String updatedResource = "entity:"+ updatedQualifiedName; + + updatedpolicyResources.put(currentResource, updatedResource); + + List policies = getPolicy(currentResource); + if (CollectionUtils.isNotEmpty(policies)) { + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(POLICY_ENTITY_TYPE); + for (AtlasEntityHeader policy : policies) { + AtlasEntity policyEntity = entityRetriever.toAtlasEntity(policy.getGuid()); + List policyResources = (List) policyEntity.getAttribute(ATTR_POLICY_RESOURCES); + policyResources.remove(currentResource); + policyResources.add(updatedResource); + AtlasVertex policyVertex = context.getVertex(policy.getGuid()); + policyVertex.removeProperty(ATTR_POLICY_RESOURCES); + policyEntity.setAttribute(ATTR_POLICY_RESOURCES, policyResources); + context.addUpdated(policyEntity.getGuid(), policyEntity, entityType, policyVertex); + } + } + + }finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + + } + + private List getPolicy(String resource) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("getPolicy"); + try { + List mustClauseList = new ArrayList(); + mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", POLICY_ENTITY_TYPE))); + mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); + mustClauseList.add(mapOf("term", mapOf("policyResources", Arrays.asList(resource)))); + + Map bool = new HashMap<>(); + bool.put("must", mustClauseList); + + Map dsl = mapOf("query", mapOf("bool", bool)); + + List policies = indexSearchPaginated(dsl); + + return policies; + } finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + } + private void setParent(AtlasEntity entity, EntityMutationContext context) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("DataProductPreProcessor.setParent"); if (parentDomain == null) { @@ -246,7 +305,7 @@ private void productExists(String productName, String parentDomainQualifiedName) } if (exists) { - throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, productName); + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, productName+" already exists in the domain"); } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index 0c2b98aa36a..a2c1618d0d7 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -33,6 +33,7 @@ import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; +import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; @@ -46,6 +47,7 @@ import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.graph.GraphHelper.getActiveChildrenVertices; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.*; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_RESOURCES; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; public class DomainPreProcessor extends AbstractDomainPreProcessor { @@ -81,7 +83,7 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co processCreateDomain(entity, vertex); break; case UPDATE: - processUpdateDomain(entity, vertex); + processUpdateDomain(entity, vertex, context); break; } } @@ -106,7 +108,7 @@ public static String createQualifiedName(String parentDomainQualifiedName) { } } - private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { + private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdateDomain"); String domainName = (String) entity.getAttribute(NAME); String vertexQnName = vertex.getProperty(QUALIFIED_NAME, String.class); @@ -143,7 +145,7 @@ private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws isAuthorized(currentParentDomainHeader, parentDomain); } - processMoveSubDomainToAnotherDomain(entity, vertex, currentParentDomainQualifiedName, newParentDomainQualifiedName, vertexQnName, superDomainQualifiedName); + processMoveSubDomainToAnotherDomain(entity, vertex, currentParentDomainQualifiedName, newParentDomainQualifiedName, vertexQnName, superDomainQualifiedName, context); } else { String vertexName = vertex.getProperty(NAME, String.class); @@ -162,7 +164,8 @@ private void processMoveSubDomainToAnotherDomain(AtlasEntity domain, String sourceDomainQualifiedName, String targetDomainQualifiedName, String currentSubDomainQualifiedName, - String superDomainQualifiedName) throws AtlasBaseException { + String superDomainQualifiedName, + EntityMutationContext context) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("processMoveSubDomainToAnotherGlossary"); try { @@ -190,7 +193,7 @@ private void processMoveSubDomainToAnotherDomain(AtlasEntity domain, domain.setAttribute(SUPER_DOMAIN_QN, superDomainQualifiedName); } - moveChildrenToAnotherDomain(domainVertex, superDomainQualifiedName, null, sourceDomainQualifiedName, targetDomainQualifiedName); + moveChildrenToAnotherDomain(domainVertex, superDomainQualifiedName, null, sourceDomainQualifiedName, targetDomainQualifiedName, context); LOG.info("Moved subDomain {} to Domain {}", domainName, targetDomainQualifiedName); @@ -203,7 +206,8 @@ private void moveChildrenToAnotherDomain(AtlasVertex childDomainVertex, String superDomainQualifiedName, String parentDomainQualifiedName, String sourceDomainQualifiedName, - String targetDomainQualifiedName) throws AtlasBaseException { + String targetDomainQualifiedName, + EntityMutationContext context) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("moveChildrenToAnotherDomain"); @@ -222,6 +226,9 @@ private void moveChildrenToAnotherDomain(AtlasVertex childDomainVertex, childDomainVertex.setProperty(SUPER_DOMAIN_QN, superDomainQualifiedName); childDomainVertex.setProperty(PARENT_DOMAIN_QN, parentDomainQualifiedName); + //update policy + updatePolicy(currentDomainQualifiedName, updatedDomainQualifiedName, context); + //update system properties GraphHelper.setModifiedByAsString(childDomainVertex, RequestContext.get().getUser()); GraphHelper.setModifiedTime(childDomainVertex, System.currentTimeMillis()); @@ -231,7 +238,7 @@ private void moveChildrenToAnotherDomain(AtlasVertex childDomainVertex, while (products.hasNext()) { AtlasVertex productVertex = products.next(); - moveChildDataProductToAnotherDomain(productVertex, superDomainQualifiedName, updatedDomainQualifiedName, sourceDomainQualifiedName, targetDomainQualifiedName); + moveChildDataProductToAnotherDomain(productVertex, superDomainQualifiedName, updatedDomainQualifiedName, sourceDomainQualifiedName, targetDomainQualifiedName, context); } // Get all children domains of current domain @@ -239,7 +246,7 @@ private void moveChildrenToAnotherDomain(AtlasVertex childDomainVertex, while (childDomains.hasNext()) { AtlasVertex childVertex = childDomains.next(); - moveChildrenToAnotherDomain(childVertex, superDomainQualifiedName, updatedDomainQualifiedName, sourceDomainQualifiedName, targetDomainQualifiedName); + moveChildrenToAnotherDomain(childVertex, superDomainQualifiedName, updatedDomainQualifiedName, sourceDomainQualifiedName, targetDomainQualifiedName, context); } recordUpdatedChildEntities(childDomainVertex, updatedAttributes); @@ -254,7 +261,8 @@ private void moveChildDataProductToAnotherDomain(AtlasVertex productVertex, String superDomainQualifiedName, String parentDomainQualifiedName, String sourceDomainQualifiedName, - String targetDomainQualifiedName) throws AtlasBaseException { + String targetDomainQualifiedName, + EntityMutationContext context) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("moveChildDataProductToAnotherDomain"); try { @@ -262,8 +270,8 @@ private void moveChildDataProductToAnotherDomain(AtlasVertex productVertex, LOG.info("Moving dataProduct {} to Domain {}", productName, targetDomainQualifiedName); Map updatedAttributes = new HashMap<>(); - String currentDomainQualifiedName = productVertex.getProperty(PARENT_DOMAIN_QN, String.class); - String updatedQualifiedName = currentDomainQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); + String currentQualifiedName = productVertex.getProperty(PARENT_DOMAIN_QN, String.class); + String updatedQualifiedName = currentQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); productVertex.setProperty(QUALIFIED_NAME, updatedQualifiedName); updatedAttributes.put(QUALIFIED_NAME, updatedQualifiedName); @@ -271,6 +279,9 @@ private void moveChildDataProductToAnotherDomain(AtlasVertex productVertex, productVertex.setProperty(PARENT_DOMAIN_QN, parentDomainQualifiedName); productVertex.setProperty(SUPER_DOMAIN_QN, superDomainQualifiedName); + //update policy + updatePolicy(currentQualifiedName, updatedQualifiedName, context); + //update system properties GraphHelper.setModifiedByAsString(productVertex, RequestContext.get().getUser()); GraphHelper.setModifiedTime(productVertex, System.currentTimeMillis()); @@ -283,6 +294,59 @@ private void moveChildDataProductToAnotherDomain(AtlasVertex productVertex, } } + private void updatePolicy(String currentQualifiedName, String updatedQualifiedName, EntityMutationContext context) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("updateDomainPolicy"); + try { + LOG.info("Updating policy for entity {}", currentQualifiedName); + Map updatedpolicyResources = new HashMap<>(); + + String currentResource = "entity:"+ currentQualifiedName; + String updatedResource = "entity:"+ updatedQualifiedName; + + updatedpolicyResources.put(currentResource, updatedResource); + + List policies = getPolicy(currentResource); + if (CollectionUtils.isNotEmpty(policies)) { + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(POLICY_ENTITY_TYPE); + for (AtlasEntityHeader policy : policies) { + AtlasEntity policyEntity = entityRetriever.toAtlasEntity(policy.getGuid()); + List policyResources = (List) policyEntity.getAttribute(ATTR_POLICY_RESOURCES); + policyResources.remove(currentResource); + policyResources.add(updatedResource); + AtlasVertex policyVertex = context.getVertex(policy.getGuid()); + policyVertex.removeProperty(ATTR_POLICY_RESOURCES); + policyEntity.setAttribute(ATTR_POLICY_RESOURCES, policyResources); + context.addUpdated(policyEntity.getGuid(), policyEntity, entityType, policyVertex); + } + } + + }finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + + } + + private List getPolicy(String resource) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("getPolicy"); + try { + List mustClauseList = new ArrayList(); + mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", POLICY_ENTITY_TYPE))); + mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); + mustClauseList.add(mapOf("term", mapOf("policyResources", Arrays.asList(resource)))); + + Map bool = new HashMap<>(); + bool.put("must", mustClauseList); + + Map dsl = mapOf("query", mapOf("bool", bool)); + + List policies = indexSearchPaginated(dsl); + + return policies; + } finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + } + private void setParent(AtlasEntity entity, EntityMutationContext context) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("DomainPreProcessor.setParent"); if (parentDomain == null) { @@ -348,7 +412,7 @@ private void domainExists(String domainName, String parentDomainQualifiedName) t } if (exists) { - throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, domainName); + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, domainName+" already exists"); } } } From c79dfa065fe0fdadd9969f9a5d298eb9fa429986 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Tue, 16 Apr 2024 15:28:19 +0530 Subject: [PATCH 034/316] Fix audit logs for associated asset --- .../store/graph/v2/AtlasEntityStoreV2.java | 2 +- .../contract/ContractPreProcessor.java | 38 +++++++++++++++---- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 2b2c6a7a3c9..ed24f329c61 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -1839,7 +1839,7 @@ public PreProcessor getPreProcessor(String typeName) { break; case CONTRACT_ENTITY_TYPE: - preProcessor = new ContractPreProcessor(graph, typeRegistry, entityRetriever, this, entityGraphMapper); + preProcessor = new ContractPreProcessor(graph, typeRegistry, entityRetriever, this, entityGraphMapper, storeDifferentialAudits); break; } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index fd82d9edd04..b64c151df78 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -34,6 +34,8 @@ public class ContractPreProcessor extends AbstractContractPreProcessor { public static final String ATTR_ASSET_QUALIFIED_NAME = "contractAssetQualifiedName"; public static final String ATTR_PARENT_GUID = "parentGuid"; public static final String ASSET_ATTR_HAS_CONTRACT = "hasContract"; + public static final String ASSET_ATTR_CONTRACT_VERSION_QUALIFIED_NAME = "latestContractQualifiedName"; + public static final String ASSET_ATTR_DESCRIPTION = "description"; public static final String CONTRACT_QUALIFIED_NAME_SUFFIX = "contract"; @@ -41,12 +43,14 @@ public class ContractPreProcessor extends AbstractContractPreProcessor { public static final String CONTRACT_ATTR_STATUS = "status"; private final AtlasEntityStore entityStore; private final EntityGraphMapper entityGraphMapper; + private boolean storeDifferentialAudits; public ContractPreProcessor(AtlasGraph graph, AtlasTypeRegistry typeRegistry, - EntityGraphRetriever entityRetriever, AtlasEntityStore entityStore, EntityGraphMapper entityGraphMapper) { + EntityGraphRetriever entityRetriever, AtlasEntityStore entityStore, EntityGraphMapper entityGraphMapper, boolean storeDifferentialAudits) { super(graph, typeRegistry, entityRetriever); + this.storeDifferentialAudits = storeDifferentialAudits; this.entityStore = entityStore; this.entityGraphMapper = entityGraphMapper; } @@ -236,19 +240,37 @@ private void contractAttributeSync(AtlasEntity entity, DataContract contract) th } private void datasetAttributeSync(EntityMutationContext context, AtlasEntity associatedAsset, DataContract contract, AtlasEntity contractAsset) throws AtlasBaseException { - associatedAsset.setAttribute(ASSET_ATTR_HAS_CONTRACT, true); + // Creating new empty AtlasEntity to update with selective attributes only + AtlasEntity entity = new AtlasEntity(associatedAsset.getTypeName()); + entity.setGuid(associatedAsset.getGuid()); + entity.setAttribute(QUALIFIED_NAME, associatedAsset.getAttribute(QUALIFIED_NAME)); + if (!associatedAsset.getAttribute(ASSET_ATTR_HAS_CONTRACT).equals(true)) { + entity.setAttribute(ASSET_ATTR_HAS_CONTRACT, true); + } + entity.setAttribute(ASSET_ATTR_CONTRACT_VERSION_QUALIFIED_NAME, contractAsset.getAttribute(QUALIFIED_NAME)); if (contract.getStatus() == DataContract.STATUS.VERIFIED && contractAsset.getAttribute(ATTR_CERTIFICATE_STATUS).equals(DataContract.STATUS.VERIFIED.name())) { DataContract.Dataset dataset = contract.dataset; // Will implement dataset attribute sync from the contract attributes - if (!dataset.description.isEmpty()) { - associatedAsset.setAttribute(ASSET_ATTR_DESCRIPTION, dataset.description); - } + // if (!dataset.description.isEmpty()) { + // associatedAsset.setAttribute(ASSET_ATTR_DESCRIPTION, dataset.description); + // } } try { - AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(associatedAsset.getGuid()); - AtlasEntityType entityType = ensureEntityType(associatedAsset.getTypeName()); - context.addUpdated(associatedAsset.getGuid(), associatedAsset, entityType, vertex); + AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(entity.getGuid()); + AtlasEntityType entityType = ensureEntityType(entity.getTypeName()); + AtlasEntityComparator entityComparator = new AtlasEntityComparator(typeRegistry, entityRetriever, context.getGuidAssignments(), true, true); + AtlasEntityComparator.AtlasEntityDiffResult diffResult = entityComparator.getDiffResult(entity, vertex, !storeDifferentialAudits); + RequestContext reqContext = RequestContext.get(); + + if (diffResult.hasDifference()) { + context.addUpdated(entity.getGuid(), entity, entityType, vertex); + if (storeDifferentialAudits) { + diffResult.getDiffEntity().setGuid(entity.getGuid()); + reqContext.cacheDifferentialEntity(diffResult.getDiffEntity()); + } + } + // RequestContext.get().setSkipAuthorizationCheck(true); // EntityStream entityStream = new AtlasEntityStream(associatedAsset); // entityStore.createOrUpdate(entityStream, false); From b09487525b566bd4bfa35acadeec68dc140fa2bd Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Tue, 16 Apr 2024 17:41:00 +0530 Subject: [PATCH 035/316] indexSearch query issue resolved --- .../graph/v2/preprocessor/datamesh/DomainPreProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index a2c1618d0d7..a33150e607a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -332,7 +332,7 @@ private List getPolicy(String resource) throws AtlasBaseExcep List mustClauseList = new ArrayList(); mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", POLICY_ENTITY_TYPE))); mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); - mustClauseList.add(mapOf("term", mapOf("policyResources", Arrays.asList(resource)))); + mustClauseList.add(mapOf("terms", mapOf("policyResources", Arrays.asList(resource)))); Map bool = new HashMap<>(); bool.put("must", mustClauseList); From 723286a517402ed120445e1634bff3fe95e0ff39 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Tue, 16 Apr 2024 17:43:55 +0530 Subject: [PATCH 036/316] Update dataset spec structure --- .../contract/ContractPreProcessor.java | 4 +- .../preprocessor/contract/DataContract.java | 87 +++++++------------ 2 files changed, 34 insertions(+), 57 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index b64c151df78..c4e75baf466 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -99,8 +99,7 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con String contractString = (String) entity.getAttribute(ATTR_CONTRACT); DataContract contract = DataContract.deserialize(contractString); - DataContract.Dataset dataset = contract.dataset; - AtlasEntityWithExtInfo associatedAsset = getAssociatedAsset(contractQName, dataset.type.name()); + AtlasEntityWithExtInfo associatedAsset = getAssociatedAsset(contractQName, contract.type.name()); authorizeContractCreateOrUpdate(entity, associatedAsset); @@ -250,7 +249,6 @@ private void datasetAttributeSync(EntityMutationContext context, AtlasEntity ass entity.setAttribute(ASSET_ATTR_CONTRACT_VERSION_QUALIFIED_NAME, contractAsset.getAttribute(QUALIFIED_NAME)); if (contract.getStatus() == DataContract.STATUS.VERIFIED && contractAsset.getAttribute(ATTR_CERTIFICATE_STATUS).equals(DataContract.STATUS.VERIFIED.name())) { - DataContract.Dataset dataset = contract.dataset; // Will implement dataset attribute sync from the contract attributes // if (!dataset.description.isEmpty()) { // associatedAsset.setAttribute(ASSET_ATTR_DESCRIPTION, dataset.description); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java index 4e0887320c5..18413f06cd8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java @@ -21,7 +21,7 @@ @JsonIgnoreProperties(ignoreUnknown = true) -@JsonPropertyOrder({"kind", "status", "template_version", "dataset", "columns"}) +@JsonPropertyOrder({"kind", "status", "template_version", "dataset", "type", "columns"}) public class DataContract { @Valid @NotNull public String kind; @@ -31,10 +31,41 @@ public class DataContract { @JsonProperty(value = "template_version", defaultValue = "0.0.1") public String templateVersion; @Valid @NotNull - public Dataset dataset; + public String dataset; + @Valid @NotNull + public DATASET_TYPE type; @Valid public List columns; private Map unknownFields = new HashMap<>(); + + @JsonSetter("type") + public void setType(String type) throws AtlasBaseException { + try { + this.type = DATASET_TYPE.from(type); + } catch (IllegalArgumentException | AtlasBaseException ex) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "type " + type + " is inappropriate. Accepted values: " + Arrays.toString(DATASET_TYPE.values())); + } + } + + public enum DATASET_TYPE { + @JsonProperty("Table") Table, + @JsonProperty("View") View, + @JsonProperty("MaterialisedView") MaterialisedView; + + public static DATASET_TYPE from(String s) throws AtlasBaseException { + + switch (s.toLowerCase()) { + case "table": + return Table; + case "view": + return View; + case "materialisedview": + return MaterialisedView; + default: + throw new AtlasBaseException(String.format("dataset.type: %s value not supported yet.", s)); + } + } + } public STATUS getStatus() { return status; } @@ -100,58 +131,6 @@ private boolean isSemVer(String version) { return matcher.matches(); } - @JsonIgnoreProperties(ignoreUnknown = true) - @JsonPropertyOrder({"name", "type", "description"}) - public static final class Dataset { - @NotNull - public String name; - @NotNull - public DATASET_TYPE type; - public String description; - private Map unknownFields = new HashMap<>(); - - @JsonAnySetter - public void setUnknownFields(String key, Object value) { - unknownFields.put(key, value); - } - - @JsonAnyGetter - public Map getUnknownFields() { - return unknownFields; - } - - @JsonSetter("type") - public void setType(String type) throws AtlasBaseException { - try { - this.type = DATASET_TYPE.from(type); - } catch (IllegalArgumentException | AtlasBaseException ex) { - throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "dataset.type " + type + " is inappropriate. Accepted values: " + Arrays.toString(DATASET_TYPE.values())); - } - } - - public enum DATASET_TYPE { - @JsonProperty("Table") Table, - @JsonProperty("View") View, - @JsonProperty("MaterialisedView") MaterialisedView; - - public static DATASET_TYPE from(String s) throws AtlasBaseException { - - switch (s.toLowerCase()) { - case "table": - return Table; - case "view": - return View; - case "materialisedview": - return MaterialisedView; - default: - throw new AtlasBaseException(String.format("dataset.type: %s value not supported yet.", s)); - } - } - } - - - } - @JsonIgnoreProperties(ignoreUnknown = true) @JsonPropertyOrder({"name", "description", "data_type"}) public static final class Column { From 3e6f1b2931c54910cf80d443dfc6b0605bd6018f Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Tue, 16 Apr 2024 18:07:43 +0530 Subject: [PATCH 037/316] Minor bug fix --- .../graph/v2/preprocessor/contract/ContractPreProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index c4e75baf466..e7c165a3285 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -243,7 +243,7 @@ private void datasetAttributeSync(EntityMutationContext context, AtlasEntity ass AtlasEntity entity = new AtlasEntity(associatedAsset.getTypeName()); entity.setGuid(associatedAsset.getGuid()); entity.setAttribute(QUALIFIED_NAME, associatedAsset.getAttribute(QUALIFIED_NAME)); - if (!associatedAsset.getAttribute(ASSET_ATTR_HAS_CONTRACT).equals(true)) { + if (associatedAsset.getAttribute(ASSET_ATTR_HAS_CONTRACT) == null || associatedAsset.getAttribute(ASSET_ATTR_HAS_CONTRACT).equals(false)) { entity.setAttribute(ASSET_ATTR_HAS_CONTRACT, true); } entity.setAttribute(ASSET_ATTR_CONTRACT_VERSION_QUALIFIED_NAME, contractAsset.getAttribute(QUALIFIED_NAME)); From 3115e484353733a3e99cfff18efacabd16ec3a5e Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Wed, 17 Apr 2024 17:29:36 +0530 Subject: [PATCH 038/316] corrected indexsearch for policy fetch --- .../datamesh/AbstractDomainPreProcessor.java | 17 ++++++++++++----- .../datamesh/DataProductPreProcessor.java | 4 ++-- .../datamesh/DomainPreProcessor.java | 4 ++-- 3 files changed, 16 insertions(+), 9 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 546d6e63bcd..d92bd688f56 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -39,11 +39,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; +import java.util.*; -import static org.apache.atlas.repository.Constants.NAME; +import static org.apache.atlas.repository.Constants.*; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.PARENT_DOMAIN_QN; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.SUPER_DOMAIN_QN; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_CATEGORY; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_RESOURCES; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; public abstract class AbstractDomainPreProcessor implements PreProcessor { @@ -66,7 +68,7 @@ public abstract class AbstractDomainPreProcessor implements PreProcessor { } } - public List indexSearchPaginated(Map dsl) throws AtlasBaseException { + public List indexSearchPaginated(Map dsl, String entityType) throws AtlasBaseException { IndexSearchParams searchParams = new IndexSearchParams(); List ret = new ArrayList<>(); @@ -83,6 +85,11 @@ public List indexSearchPaginated(Map dsl) thr dsl.put("size", size); searchParams.setDsl(dsl); + if (entityType.equals(POLICY_ENTITY_TYPE)) { + Set attributes = new HashSet<>(Arrays.asList(ATTR_POLICY_RESOURCES, ATTR_POLICY_CATEGORY)); + searchParams.setAttributes(attributes); + } + List headers = discovery.directIndexSearch(searchParams).getEntities(); if (CollectionUtils.isNotEmpty(headers)) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 057de37acac..ea7905ae8e4 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -206,7 +206,7 @@ private List getPolicy(String resource) throws AtlasBaseExcep Map dsl = mapOf("query", mapOf("bool", bool)); - List policies = indexSearchPaginated(dsl); + List policies = indexSearchPaginated(dsl, POLICY_ENTITY_TYPE); return policies; } finally { @@ -289,7 +289,7 @@ private void productExists(String productName, String parentDomainQualifiedName) Map dsl = mapOf("query", mapOf("bool", bool)); - List products = indexSearchPaginated(dsl); + List products = indexSearchPaginated(dsl, DATA_PRODUCT_ENTITY_TYPE); if (CollectionUtils.isNotEmpty(products)) { for (AtlasEntityHeader product : products) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index a33150e607a..e944a638800 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -339,7 +339,7 @@ private List getPolicy(String resource) throws AtlasBaseExcep Map dsl = mapOf("query", mapOf("bool", bool)); - List policies = indexSearchPaginated(dsl); + List policies = indexSearchPaginated(dsl, POLICY_ENTITY_TYPE); return policies; } finally { @@ -396,7 +396,7 @@ private void domainExists(String domainName, String parentDomainQualifiedName) t Map dsl = mapOf("query", mapOf("bool", bool)); - List domains = indexSearchPaginated(dsl); + List domains = indexSearchPaginated(dsl, DATA_DOMAIN_ENTITY_TYPE); if (CollectionUtils.isNotEmpty(domains)) { for (AtlasEntityHeader domain : domains) { From fd04fc90ffe521ef8a1e71109368c255cbfedfe1 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Thu, 18 Apr 2024 18:17:32 +0530 Subject: [PATCH 039/316] Update maven.yml --- .github/workflows/maven.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 0977cb36a25..b7b18d56b98 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -21,11 +21,10 @@ name: Java CI with Maven on: push: branches: - - alpha + - staging - beta - development - master - - lineageondemand jobs: build: From 084ac61ee5e060c062d0239173c881879bdb7171 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Mon, 22 Apr 2024 13:20:17 +0530 Subject: [PATCH 040/316] logs added to debug policy issue --- .../datamesh/AbstractDomainPreProcessor.java | 62 ++++++++++++- .../datamesh/DataProductPreProcessor.java | 87 +++---------------- .../datamesh/DomainPreProcessor.java | 82 +++-------------- 3 files changed, 84 insertions(+), 147 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index d92bd688f56..9ff2ed5d074 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -30,6 +30,7 @@ import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.glossary.AbstractGlossaryPreProcessor; import org.apache.atlas.type.AtlasEntityType; @@ -42,14 +43,12 @@ import java.util.*; import static org.apache.atlas.repository.Constants.*; -import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.PARENT_DOMAIN_QN; -import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.SUPER_DOMAIN_QN; import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_CATEGORY; import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_RESOURCES; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; public abstract class AbstractDomainPreProcessor implements PreProcessor { - private static final Logger LOG = LoggerFactory.getLogger(AbstractGlossaryPreProcessor.class); + private static final Logger LOG = LoggerFactory.getLogger(AbstractDomainPreProcessor.class); protected final AtlasTypeRegistry typeRegistry; @@ -129,6 +128,63 @@ protected void isAuthorized(AtlasEntityHeader sourceDomain, AtlasEntityHeader ta "delete on target Domain: ", targetDomain.getAttribute(NAME)); } + protected void updatePolicy(String currentQualifiedName, String updatedQualifiedName, EntityMutationContext context) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("updateDomainPolicy"); + try { + LOG.info("Updating policy for entity {}", currentQualifiedName); + Map updatedpolicyResources = new HashMap<>(); + + String currentResource = "entity:"+ currentQualifiedName; + String updatedResource = "entity:"+ updatedQualifiedName; + + updatedpolicyResources.put(currentResource, updatedResource); + + List policies = getPolicy(currentResource); + if (CollectionUtils.isNotEmpty(policies)) { + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(POLICY_ENTITY_TYPE); + for (AtlasEntityHeader policy : policies) { + LOG.info("Updating policy {} for entity {}", policy.getGuid(), currentQualifiedName); + AtlasEntity policyEntity = entityRetriever.toAtlasEntity(policy.getGuid()); + List policyResources = (List) policyEntity.getAttribute(ATTR_POLICY_RESOURCES); + LOG.info("Policy resources {}", policyResources); + policyResources.remove(currentResource); + policyResources.add(updatedResource); + AtlasVertex policyVertex = context.getVertex(policy.getGuid()); + LOG.info("Policy Vertex {}", policyVertex); + LOG.info("Context {}", context); + policyVertex.removeProperty(ATTR_POLICY_RESOURCES); + policyEntity.setAttribute(ATTR_POLICY_RESOURCES, policyResources); + context.addUpdated(policyEntity.getGuid(), policyEntity, entityType, policyVertex); + } + } + + }finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + + } + + protected List getPolicy(String resource) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("getPolicy"); + try { + List mustClauseList = new ArrayList(); + mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", POLICY_ENTITY_TYPE))); + mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); + mustClauseList.add(mapOf("terms", mapOf("policyResources", Arrays.asList(resource)))); + + Map bool = new HashMap<>(); + bool.put("must", mustClauseList); + + Map dsl = mapOf("query", mapOf("bool", bool)); + + List policies = indexSearchPaginated(dsl, POLICY_ENTITY_TYPE); + + return policies; + } finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + } + /** * Record the updated child entities, it will be used to send notification and store audit logs * @param entityVertex Child entity vertex diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index ea7905ae8e4..14b0b6d0fab 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -22,7 +22,6 @@ import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.*; -import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_RESOURCES; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; public class DataProductPreProcessor extends AbstractDomainPreProcessor { @@ -54,38 +53,32 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co processCreateProduct(entity, vertex); break; case UPDATE: - processUpdateDomain(entity, vertex, context); + processUpdateDomain(entity, vertex); break; } } private void processCreateProduct(AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { - AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreateDomain"); - String domainName = (String) entity.getAttribute(NAME); + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreateProduct"); + String productName = (String) entity.getAttribute(NAME); String parentDomainQualifiedName = (String) entity.getAttribute(PARENT_DOMAIN_QN); - productExists(domainName, parentDomainQualifiedName); + productExists(productName, parentDomainQualifiedName); String newQualifiedName = createQualifiedName(parentDomainQualifiedName); - if(!newQualifiedName.isEmpty()){ - entity.setAttribute(QUALIFIED_NAME, newQualifiedName); - } - else{ - throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Parent Domain Qualified Name is empty"); - } + + entity.setAttribute(QUALIFIED_NAME, newQualifiedName); RequestContext.get().endMetricRecord(metricRecorder); } - public static String createQualifiedName(String parentDomainQualifiedName) { - if (StringUtils.isNotEmpty(parentDomainQualifiedName) && parentDomainQualifiedName !=null) { - return parentDomainQualifiedName + "/product/" + getUUID(); - } - else{ - return ""; + private static String createQualifiedName(String parentDomainQualifiedName) throws AtlasBaseException { + if (StringUtils.isNotEmpty(parentDomainQualifiedName)) { + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Parent Domain Qualified Name cannot be empty or null"); } + return parentDomainQualifiedName + "/product/" + getUUID(); } - private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) throws AtlasBaseException { + private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdateDomain"); String productName = (String) entity.getAttribute(NAME); String vertexQnName = vertex.getProperty(QUALIFIED_NAME, String.class); @@ -116,7 +109,7 @@ private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex, EntityM //Auth check isAuthorized(currentParentDomainHeader, parentDomain); - processMoveDataProductToAnotherDomain(entity, currentParentDomainQualifiedName, newParentDomainQualifiedName, vertexQnName, superDomainQualifiedName, context); + processMoveDataProductToAnotherDomain(entity, currentParentDomainQualifiedName, newParentDomainQualifiedName, vertexQnName, superDomainQualifiedName); entity.setAttribute(PARENT_DOMAIN_QN, newParentDomainQualifiedName); } else { @@ -134,8 +127,7 @@ private void processMoveDataProductToAnotherDomain(AtlasEntity product, String sourceDomainQualifiedName, String targetDomainQualifiedName, String currentDataProductQualifiedName, - String superDomainQualifiedName, - EntityMutationContext context) throws AtlasBaseException { + String superDomainQualifiedName) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("processMoveDataProductToAnotherDomain"); try { @@ -161,59 +153,6 @@ private void processMoveDataProductToAnotherDomain(AtlasEntity product, } } - private void updatePolicy(String currentQualifiedName, String updatedQualifiedName, EntityMutationContext context) throws AtlasBaseException { - AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("updateDomainPolicy"); - try { - LOG.info("Updating policy for entity {}", currentQualifiedName); - Map updatedpolicyResources = new HashMap<>(); - - String currentResource = "entity:"+ currentQualifiedName; - String updatedResource = "entity:"+ updatedQualifiedName; - - updatedpolicyResources.put(currentResource, updatedResource); - - List policies = getPolicy(currentResource); - if (CollectionUtils.isNotEmpty(policies)) { - AtlasEntityType entityType = typeRegistry.getEntityTypeByName(POLICY_ENTITY_TYPE); - for (AtlasEntityHeader policy : policies) { - AtlasEntity policyEntity = entityRetriever.toAtlasEntity(policy.getGuid()); - List policyResources = (List) policyEntity.getAttribute(ATTR_POLICY_RESOURCES); - policyResources.remove(currentResource); - policyResources.add(updatedResource); - AtlasVertex policyVertex = context.getVertex(policy.getGuid()); - policyVertex.removeProperty(ATTR_POLICY_RESOURCES); - policyEntity.setAttribute(ATTR_POLICY_RESOURCES, policyResources); - context.addUpdated(policyEntity.getGuid(), policyEntity, entityType, policyVertex); - } - } - - }finally { - RequestContext.get().endMetricRecord(metricRecorder); - } - - } - - private List getPolicy(String resource) throws AtlasBaseException { - AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("getPolicy"); - try { - List mustClauseList = new ArrayList(); - mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", POLICY_ENTITY_TYPE))); - mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); - mustClauseList.add(mapOf("term", mapOf("policyResources", Arrays.asList(resource)))); - - Map bool = new HashMap<>(); - bool.put("must", mustClauseList); - - Map dsl = mapOf("query", mapOf("bool", bool)); - - List policies = indexSearchPaginated(dsl, POLICY_ENTITY_TYPE); - - return policies; - } finally { - RequestContext.get().endMetricRecord(metricRecorder); - } - } - private void setParent(AtlasEntity entity, EntityMutationContext context) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("DataProductPreProcessor.setParent"); if (parentDomain == null) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index e944a638800..abe27701e03 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -47,7 +47,6 @@ import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.graph.GraphHelper.getActiveChildrenVertices; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.*; -import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_RESOURCES; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; public class DomainPreProcessor extends AbstractDomainPreProcessor { @@ -83,7 +82,7 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co processCreateDomain(entity, vertex); break; case UPDATE: - processUpdateDomain(entity, vertex, context); + processUpdateDomain(entity, vertex); break; } } @@ -99,16 +98,15 @@ private void processCreateDomain(AtlasEntity entity, AtlasVertex vertex) throws RequestContext.get().endMetricRecord(metricRecorder); } - public static String createQualifiedName(String parentDomainQualifiedName) { - if (StringUtils.isNotEmpty(parentDomainQualifiedName) && parentDomainQualifiedName !=null) { + private static String createQualifiedName(String parentDomainQualifiedName) { + if (StringUtils.isNotEmpty(parentDomainQualifiedName)) { return parentDomainQualifiedName + "/domain/" + getUUID(); } else{ - String prefixQN = "default/domain"; - return prefixQN + "/" + getUUID(); + return "default/domain" + "/" + getUUID(); } } - private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex, EntityMutationContext context) throws AtlasBaseException { + private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdateDomain"); String domainName = (String) entity.getAttribute(NAME); String vertexQnName = vertex.getProperty(QUALIFIED_NAME, String.class); @@ -145,7 +143,7 @@ private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex, EntityM isAuthorized(currentParentDomainHeader, parentDomain); } - processMoveSubDomainToAnotherDomain(entity, vertex, currentParentDomainQualifiedName, newParentDomainQualifiedName, vertexQnName, superDomainQualifiedName, context); + processMoveSubDomainToAnotherDomain(entity, vertex, currentParentDomainQualifiedName, newParentDomainQualifiedName, vertexQnName, superDomainQualifiedName); } else { String vertexName = vertex.getProperty(NAME, String.class); @@ -164,8 +162,7 @@ private void processMoveSubDomainToAnotherDomain(AtlasEntity domain, String sourceDomainQualifiedName, String targetDomainQualifiedName, String currentSubDomainQualifiedName, - String superDomainQualifiedName, - EntityMutationContext context) throws AtlasBaseException { + String superDomainQualifiedName) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("processMoveSubDomainToAnotherGlossary"); try { @@ -193,7 +190,7 @@ private void processMoveSubDomainToAnotherDomain(AtlasEntity domain, domain.setAttribute(SUPER_DOMAIN_QN, superDomainQualifiedName); } - moveChildrenToAnotherDomain(domainVertex, superDomainQualifiedName, null, sourceDomainQualifiedName, targetDomainQualifiedName, context); + moveChildrenToAnotherDomain(domainVertex, superDomainQualifiedName, null, sourceDomainQualifiedName, targetDomainQualifiedName); LOG.info("Moved subDomain {} to Domain {}", domainName, targetDomainQualifiedName); @@ -206,8 +203,7 @@ private void moveChildrenToAnotherDomain(AtlasVertex childDomainVertex, String superDomainQualifiedName, String parentDomainQualifiedName, String sourceDomainQualifiedName, - String targetDomainQualifiedName, - EntityMutationContext context) throws AtlasBaseException { + String targetDomainQualifiedName) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("moveChildrenToAnotherDomain"); @@ -238,7 +234,7 @@ private void moveChildrenToAnotherDomain(AtlasVertex childDomainVertex, while (products.hasNext()) { AtlasVertex productVertex = products.next(); - moveChildDataProductToAnotherDomain(productVertex, superDomainQualifiedName, updatedDomainQualifiedName, sourceDomainQualifiedName, targetDomainQualifiedName, context); + moveChildDataProductToAnotherDomain(productVertex, superDomainQualifiedName, updatedDomainQualifiedName, sourceDomainQualifiedName, targetDomainQualifiedName); } // Get all children domains of current domain @@ -246,7 +242,7 @@ private void moveChildrenToAnotherDomain(AtlasVertex childDomainVertex, while (childDomains.hasNext()) { AtlasVertex childVertex = childDomains.next(); - moveChildrenToAnotherDomain(childVertex, superDomainQualifiedName, updatedDomainQualifiedName, sourceDomainQualifiedName, targetDomainQualifiedName, context); + moveChildrenToAnotherDomain(childVertex, superDomainQualifiedName, updatedDomainQualifiedName, sourceDomainQualifiedName, targetDomainQualifiedName); } recordUpdatedChildEntities(childDomainVertex, updatedAttributes); @@ -261,8 +257,7 @@ private void moveChildDataProductToAnotherDomain(AtlasVertex productVertex, String superDomainQualifiedName, String parentDomainQualifiedName, String sourceDomainQualifiedName, - String targetDomainQualifiedName, - EntityMutationContext context) throws AtlasBaseException { + String targetDomainQualifiedName) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("moveChildDataProductToAnotherDomain"); try { @@ -294,59 +289,6 @@ private void moveChildDataProductToAnotherDomain(AtlasVertex productVertex, } } - private void updatePolicy(String currentQualifiedName, String updatedQualifiedName, EntityMutationContext context) throws AtlasBaseException { - AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("updateDomainPolicy"); - try { - LOG.info("Updating policy for entity {}", currentQualifiedName); - Map updatedpolicyResources = new HashMap<>(); - - String currentResource = "entity:"+ currentQualifiedName; - String updatedResource = "entity:"+ updatedQualifiedName; - - updatedpolicyResources.put(currentResource, updatedResource); - - List policies = getPolicy(currentResource); - if (CollectionUtils.isNotEmpty(policies)) { - AtlasEntityType entityType = typeRegistry.getEntityTypeByName(POLICY_ENTITY_TYPE); - for (AtlasEntityHeader policy : policies) { - AtlasEntity policyEntity = entityRetriever.toAtlasEntity(policy.getGuid()); - List policyResources = (List) policyEntity.getAttribute(ATTR_POLICY_RESOURCES); - policyResources.remove(currentResource); - policyResources.add(updatedResource); - AtlasVertex policyVertex = context.getVertex(policy.getGuid()); - policyVertex.removeProperty(ATTR_POLICY_RESOURCES); - policyEntity.setAttribute(ATTR_POLICY_RESOURCES, policyResources); - context.addUpdated(policyEntity.getGuid(), policyEntity, entityType, policyVertex); - } - } - - }finally { - RequestContext.get().endMetricRecord(metricRecorder); - } - - } - - private List getPolicy(String resource) throws AtlasBaseException { - AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("getPolicy"); - try { - List mustClauseList = new ArrayList(); - mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", POLICY_ENTITY_TYPE))); - mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); - mustClauseList.add(mapOf("terms", mapOf("policyResources", Arrays.asList(resource)))); - - Map bool = new HashMap<>(); - bool.put("must", mustClauseList); - - Map dsl = mapOf("query", mapOf("bool", bool)); - - List policies = indexSearchPaginated(dsl, POLICY_ENTITY_TYPE); - - return policies; - } finally { - RequestContext.get().endMetricRecord(metricRecorder); - } - } - private void setParent(AtlasEntity entity, EntityMutationContext context) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("DomainPreProcessor.setParent"); if (parentDomain == null) { From 2a8c883e3f4e9b3bfb322a4d90332e2f284f11db Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Mon, 22 Apr 2024 16:51:59 +0530 Subject: [PATCH 041/316] policy vertex fetch logic changed --- .../v2/preprocessor/datamesh/AbstractDomainPreProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 9ff2ed5d074..985ecb786d5 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -149,7 +149,7 @@ protected void updatePolicy(String currentQualifiedName, String updatedQualified LOG.info("Policy resources {}", policyResources); policyResources.remove(currentResource); policyResources.add(updatedResource); - AtlasVertex policyVertex = context.getVertex(policy.getGuid()); + AtlasVertex policyVertex = entityRetriever.getEntityVertex(policy.getGuid()); LOG.info("Policy Vertex {}", policyVertex); LOG.info("Context {}", context); policyVertex.removeProperty(ATTR_POLICY_RESOURCES); From f69d6f688295666c83fc2afdf73b48ae3f1fd41c Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Mon, 22 Apr 2024 17:41:44 +0530 Subject: [PATCH 042/316] Refactor with new attributes --- .../store/graph/v2/AtlasEntityStoreV2.java | 2 +- .../AbstractContractPreProcessor.java | 16 --- .../contract/ContractPreProcessor.java | 129 +++++++----------- .../contract/ContractVersionUtils.java | 113 ++++----------- 4 files changed, 82 insertions(+), 178 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index ed24f329c61..8e11aaeff1b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -1839,7 +1839,7 @@ public PreProcessor getPreProcessor(String typeName) { break; case CONTRACT_ENTITY_TYPE: - preProcessor = new ContractPreProcessor(graph, typeRegistry, entityRetriever, this, entityGraphMapper, storeDifferentialAudits); + preProcessor = new ContractPreProcessor(graph, typeRegistry, entityRetriever, this, storeDifferentialAudits, discovery); break; } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java index dd2e558156b..4ef8d0ee6df 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java @@ -55,22 +55,6 @@ void authorizeContractCreateOrUpdate(AtlasEntity contractEntity, AtlasEntity.Atl } - void authorizeContractDelete(AtlasVertex contractVertex, String typeName) throws AtlasBaseException { - AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("authorizeContractDelete"); - - try { - AtlasEntity contractEntity = entityRetriever.toAtlasEntity(contractVertex); - String contractQName = contractEntity.getAttribute(QUALIFIED_NAME).toString(); - AtlasEntity.AtlasEntityWithExtInfo assetEntity = getAssociatedAsset(contractQName, typeName); - AtlasEntityHeader entityHeader = new AtlasEntityHeader(assetEntity.getEntity()); - - verifyAssetAccess(entityHeader, AtlasPrivilege.ENTITY_UPDATE, contractEntity, AtlasPrivilege.ENTITY_DELETE); - } finally { - RequestContext.get().endMetricRecord(recorder); - } - } - - private void verifyAssetAccess(AtlasEntityHeader asset, AtlasPrivilege assetPrivilege, AtlasEntity contract, AtlasPrivilege contractPrivilege) throws AtlasBaseException { verifyAccess(asset, assetPrivilege); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index e7c165a3285..173f0667538 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.atlas.RequestContext; +import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; @@ -26,33 +27,36 @@ import static org.apache.atlas.AtlasErrorCode.*; import static org.apache.atlas.repository.Constants.ATTR_CERTIFICATE_STATUS; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; +import static org.apache.atlas.type.AtlasTypeUtil.getAtlasObjectId; public class ContractPreProcessor extends AbstractContractPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(ContractPreProcessor.class); - public static final String ATTR_CONTRACT = "contract"; - public static final String ATTR_VERSION = "contractVersion"; - public static final String ATTR_ASSET_QUALIFIED_NAME = "contractAssetQualifiedName"; - public static final String ATTR_PARENT_GUID = "parentGuid"; + public static final String ATTR_CONTRACT = "dataContractJson"; + public static final String ATTR_VERSION = "dataContractVersion"; + public static final String REL_ATTR_GOVERNED_ASSET = "dataContractGovernedAsset"; + public static final String REL_ATTR_LATEST_CONTRACT = "dataContractLatest"; + public static final String REL_ATTR_GOVERNED_ASSET_CERTIFIED = "dataContractGovernedAssetCertified"; + public static final String REL_ATTR_PREVIOUS_VERSION = "dataContractPreviousVersion"; public static final String ASSET_ATTR_HAS_CONTRACT = "hasContract"; - public static final String ASSET_ATTR_CONTRACT_VERSION_QUALIFIED_NAME = "latestContractQualifiedName"; - public static final String ASSET_ATTR_DESCRIPTION = "description"; public static final String CONTRACT_QUALIFIED_NAME_SUFFIX = "contract"; public static final String VERSION_PREFIX = "version"; public static final String CONTRACT_ATTR_STATUS = "status"; private final AtlasEntityStore entityStore; - private final EntityGraphMapper entityGraphMapper; - private boolean storeDifferentialAudits; + private final boolean storeDifferentialAudits; + private EntityDiscoveryService discovery; + public ContractPreProcessor(AtlasGraph graph, AtlasTypeRegistry typeRegistry, - EntityGraphRetriever entityRetriever, AtlasEntityStore entityStore, EntityGraphMapper entityGraphMapper, boolean storeDifferentialAudits) { + EntityGraphRetriever entityRetriever, AtlasEntityStore entityStore, + boolean storeDifferentialAudits, EntityDiscoveryService discovery) { super(graph, typeRegistry, entityRetriever); this.storeDifferentialAudits = storeDifferentialAudits; this.entityStore = entityStore; - this.entityGraphMapper = entityGraphMapper; + this.discovery = discovery; } @Override @@ -107,50 +111,38 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con contractString = DataContract.serialize(contract); entity.setAttribute(ATTR_CONTRACT, contractString); - - ContractVersionUtils versionUtil = new ContractVersionUtils(entity, context, entityRetriever, typeRegistry, entityStore, graph); - AtlasEntity latestExistingVersion = versionUtil.getLatestVersion(); - - if (latestExistingVersion != null) { + ContractVersionUtils versionUtil = new ContractVersionUtils(entity, context, entityRetriever, typeRegistry, entityStore, graph, discovery); + AtlasEntity currentVersionEntity = versionUtil.getCurrentVersion(); + String latestVersion = "V1"; + if (currentVersionEntity != null) { // Contract already exist - String qName = (String) latestExistingVersion.getAttribute(QUALIFIED_NAME); - Integer latestVersionNumber = Integer.valueOf(qName.substring(qName.lastIndexOf("/V") + 2)); - List attributes = getDiffAttributes(context, entity, latestExistingVersion); + String qName = (String) currentVersionEntity.getAttribute(QUALIFIED_NAME); + Integer currentVersionNumber = Integer.valueOf(qName.substring(qName.lastIndexOf("/V") + 2)); + List attributes = getDiffAttributes(context, entity, currentVersionEntity); if (attributes.isEmpty()) { + // No changes in the contract, Not creating new version removeCreatingVertex(context, entity); return; - } - - if (attributes.size() == 1 && attributes.contains(ATTR_CERTIFICATE_STATUS)) { - if (Objects.equals(entity.getAttribute(ATTR_CERTIFICATE_STATUS).toString(), DataContract.STATUS.VERIFIED.name())) { - //update existing entity - updateExistingVersion(context, entity, latestExistingVersion); - } - // Contract status changed, either to draft or verified - - } else if (attributes.contains(ATTR_CONTRACT)) { - //Contract is changed - if (isEqualContract(contractString, (String) latestExistingVersion.getAttribute(ATTR_CONTRACT))) { - // Update the same asset(entity) - updateExistingVersion(context, entity, latestExistingVersion); + } else if (isEqualContract(contractString, (String) currentVersionEntity.getAttribute(ATTR_CONTRACT))) { + // No change in contract, metadata changed + updateExistingVersion(context, entity, currentVersionEntity); + return; + } else { + // contract changed (metadata might/not changed). Create new version. + latestVersion = String.format("V%s", currentVersionNumber + 1); - } else { - // Create New version of entity - entity.setAttribute(QUALIFIED_NAME, String.format("%s/%s/V%s", contractQName, VERSION_PREFIX, ++latestVersionNumber)); - entity.setAttribute(ATTR_VERSION, String.format("V%s", latestVersionNumber)); - entity.setAttribute(ATTR_ASSET_QUALIFIED_NAME, associatedAsset.getEntity().getAttribute(QUALIFIED_NAME)); - entity.setAttribute(ATTR_PARENT_GUID, latestExistingVersion.getGuid()); + // Attach previous version via rel + entity.setRelationshipAttribute(REL_ATTR_PREVIOUS_VERSION, getAtlasObjectId(currentVersionEntity)); + AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(currentVersionEntity.getGuid()); + AtlasEntityType entityType = ensureEntityType(currentVersionEntity.getTypeName()); + context.addUpdated(currentVersionEntity.getGuid(), currentVersionEntity, entityType, vertex); - } } - - } else { - // Create new contract - entity.setAttribute(QUALIFIED_NAME, String.format("%s/%s/%s", contractQName, VERSION_PREFIX, "V1")); - entity.setAttribute(ATTR_VERSION, "V1"); - entity.setAttribute(ATTR_ASSET_QUALIFIED_NAME, associatedAsset.getEntity().getAttribute(QUALIFIED_NAME)); - } + entity.setAttribute(QUALIFIED_NAME, String.format("%s/%s/%s", contractQName, VERSION_PREFIX, latestVersion)); + entity.setAttribute(ATTR_VERSION, latestVersion); + entity.setRelationshipAttribute(REL_ATTR_GOVERNED_ASSET, getAtlasObjectId(associatedAsset.getEntity())); + datasetAttributeSync(context, associatedAsset.getEntity(), contract, entity); } @@ -186,10 +178,10 @@ private boolean isEqualContract(String firstNode, String secondNode) throws Atla } - private void updateExistingVersion(EntityMutationContext context, AtlasEntity entity, AtlasEntity latestExistingVersion) throws AtlasBaseException { + private void updateExistingVersion(EntityMutationContext context, AtlasEntity entity, AtlasEntity currentVersionEntity) throws AtlasBaseException { removeCreatingVertex(context, entity); - entity.setAttribute(QUALIFIED_NAME, latestExistingVersion.getAttribute(QUALIFIED_NAME)); - entity.setGuid(latestExistingVersion.getGuid()); + entity.setAttribute(QUALIFIED_NAME, currentVersionEntity.getAttribute(QUALIFIED_NAME)); + entity.setGuid(currentVersionEntity.getGuid()); AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(entity.getGuid()); @@ -246,35 +238,18 @@ private void datasetAttributeSync(EntityMutationContext context, AtlasEntity ass if (associatedAsset.getAttribute(ASSET_ATTR_HAS_CONTRACT) == null || associatedAsset.getAttribute(ASSET_ATTR_HAS_CONTRACT).equals(false)) { entity.setAttribute(ASSET_ATTR_HAS_CONTRACT, true); } - entity.setAttribute(ASSET_ATTR_CONTRACT_VERSION_QUALIFIED_NAME, contractAsset.getAttribute(QUALIFIED_NAME)); - if (contract.getStatus() == DataContract.STATUS.VERIFIED && - contractAsset.getAttribute(ATTR_CERTIFICATE_STATUS).equals(DataContract.STATUS.VERIFIED.name())) { - // Will implement dataset attribute sync from the contract attributes - // if (!dataset.description.isEmpty()) { - // associatedAsset.setAttribute(ASSET_ATTR_DESCRIPTION, dataset.description); - // } - } - try { - AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(entity.getGuid()); - AtlasEntityType entityType = ensureEntityType(entity.getTypeName()); - AtlasEntityComparator entityComparator = new AtlasEntityComparator(typeRegistry, entityRetriever, context.getGuidAssignments(), true, true); - AtlasEntityComparator.AtlasEntityDiffResult diffResult = entityComparator.getDiffResult(entity, vertex, !storeDifferentialAudits); - RequestContext reqContext = RequestContext.get(); - - if (diffResult.hasDifference()) { - context.addUpdated(entity.getGuid(), entity, entityType, vertex); - if (storeDifferentialAudits) { - diffResult.getDiffEntity().setGuid(entity.getGuid()); - reqContext.cacheDifferentialEntity(diffResult.getDiffEntity()); - } - } + AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(entity.getGuid()); + AtlasEntityType entityType = ensureEntityType(entity.getTypeName()); + AtlasEntityComparator entityComparator = new AtlasEntityComparator(typeRegistry, entityRetriever, context.getGuidAssignments(), true, true); + AtlasEntityComparator.AtlasEntityDiffResult diffResult = entityComparator.getDiffResult(entity, vertex, !storeDifferentialAudits); + RequestContext reqContext = RequestContext.get(); + context.addUpdated(entity.getGuid(), entity, entityType, vertex); -// RequestContext.get().setSkipAuthorizationCheck(true); -// EntityStream entityStream = new AtlasEntityStream(associatedAsset); -// entityStore.createOrUpdate(entityStream, false); -// LOG.info("Updated associated asset attributes of contract {}", associatedAsset.getAttribute(QUALIFIED_NAME)); - } finally { -// RequestContext.get().setSkipAuthorizationCheck(false); + if (diffResult.hasDifference()) { + if (storeDifferentialAudits) { + diffResult.getDiffEntity().setGuid(entity.getGuid()); + reqContext.cacheDifferentialEntity(diffResult.getDiffEntity()); + } } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java index 8d27a10bc1b..3f93d305e91 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java @@ -1,29 +1,23 @@ package org.apache.atlas.repository.store.graph.v2.preprocessor.contract; -import org.apache.atlas.AtlasErrorCode; -import org.apache.atlas.RequestContext; +import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.discovery.AtlasSearchResult; +import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.repository.graphdb.AtlasGraph; -import org.apache.atlas.repository.graphdb.AtlasGraphQuery; -import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.AtlasEntityStore; import org.apache.atlas.repository.store.graph.v2.*; -import org.apache.atlas.repository.store.graph.v2.preprocessor.ConnectionPreProcessor; -import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; -import org.apache.atlas.utils.AtlasPerfMetrics; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.*; import static org.apache.atlas.repository.Constants.*; -import static org.apache.atlas.repository.graph.AtlasGraphProvider.getGraphInstance; +import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; public class ContractVersionUtils { - private static final Logger LOG = LoggerFactory.getLogger(ConnectionPreProcessor.class); private final EntityMutationContext context; public final EntityGraphRetriever entityRetriever; @@ -33,58 +27,46 @@ public class ContractVersionUtils { private AtlasEntity entity; public final AtlasGraph graph; - private List versionList; - - + private List versionList; + private EntityDiscoveryService discovery; public ContractVersionUtils(AtlasEntity entity, EntityMutationContext context, EntityGraphRetriever entityRetriever, - AtlasTypeRegistry atlasTypeRegistry, AtlasEntityStore entityStore, AtlasGraph graph) { + AtlasTypeRegistry atlasTypeRegistry, AtlasEntityStore entityStore, AtlasGraph graph, + EntityDiscoveryService discovery) { this.context = context; this.entityRetriever = entityRetriever; this.atlasTypeRegistry = atlasTypeRegistry; this.graph = graph; this.entityStore = entityStore; this.entity = entity; + this.discovery = discovery; } - private void extractAllVersions() { + private void extractAllVersions() throws AtlasBaseException { String contractQName = (String) entity.getAttribute(QUALIFIED_NAME); String datasetQName = contractQName.substring(0, contractQName.lastIndexOf("/contract")); + List ret = new ArrayList<>(); - AtlasEntityType entityType = atlasTypeRegistry.getEntityTypeByName("DataContract"); - Integer versionCounter = 1; - boolean found = true; - - List versionList = new ArrayList<>(); - - while (found) { - Map uniqAttributes = new HashMap<>(); - uniqAttributes.put(QUALIFIED_NAME, String.format("%s/contract/version/V%s", datasetQName, versionCounter++)); - try { - AtlasVertex entityVertex = AtlasGraphUtilsV2.getVertexByUniqueAttributes(graph, entityType, uniqAttributes); - AtlasEntity entity = entityRetriever.toAtlasEntity(entityVertex); -// -// EntityGraphRetriever entityRetriever = new EntityGraphRetriever(graph, typeRegistry, true); -// -// AtlasEntity.AtlasEntityWithExtInfo ret = entityRetriever.toAtlasEntityWithExtInfo(entityVertex); -// -// if (ret == null) { -// throw new AtlasBaseException(AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND, entityType.getTypeName(), -// uniqAttributes.toString()); -// } -// return ret; - - versionList.add(entity); - } catch (AtlasBaseException ex) { - found = false; - } + IndexSearchParams indexSearchParams = new IndexSearchParams(); + Map dsl = new HashMap<>(); - } - this.versionList = versionList; + List mustClauseList = new ArrayList(); + mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", CONTRACT_ENTITY_TYPE))); + mustClauseList.add(mapOf("wildcard", mapOf(QUALIFIED_NAME, String.format("%s/contract/version/*", datasetQName)))); + + dsl.put("query", mapOf("bool", mapOf("must", mustClauseList))); + indexSearchParams.setDsl(dsl); + indexSearchParams.setSuppressLogs(true); + + AtlasSearchResult result = discovery.directIndexSearch(indexSearchParams); + if (result != null) { + ret = result.getEntities(); + } + this.versionList = ret; } - public AtlasEntity getLatestVersion() throws AtlasBaseException { + public AtlasEntity getCurrentVersion() throws AtlasBaseException { if (this.versionList == null) { extractAllVersions(); } @@ -97,43 +79,6 @@ public AtlasEntity getLatestVersion() throws AtlasBaseException { if (this.versionList.isEmpty()) { return null; } - return this.versionList.get(0); - } - - - public Iterator getAllEntityVersions() { - String entityQNamePrefix = (String) entity.getAttribute(QUALIFIED_NAME); - AtlasEntityType entityType = atlasTypeRegistry.getEntityTypeByName("DataContract"); - -// AtlasEntityType entityType, String name - AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("getAllEntityVersions"); - AtlasGraph graph = getGraphInstance(); - AtlasGraphQuery query = graph.query() - .has(ENTITY_TYPE_PROPERTY_KEY, entityType.getTypeName()) - .has(STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name()) - .has(entityType.getAllAttributes().get(QUALIFIED_NAME).getQualifiedName(), String.format("%s/version/V1", entityQNamePrefix)); - - - Iterator result = query.vertices().iterator(); - - RequestContext.get().endMetricRecord(metric); - return result; + return new AtlasEntity(this.versionList.get(0)); } - -// public void createNewVersion() throws AtlasBaseException { -// AtlasVertex vertex = context.getVertex(entity.getGuid()); -// AtlasEntity existingContractEntity = entityRetriever.toAtlasEntity(vertex); -//// this.newEntity = new AtlasEntity(existingEntity); -// this.existingEntity.setAttribute(QUALIFIED_NAME, null); -// -// try { -// RequestContext.get().setSkipAuthorizationCheck(true); -// EntityStream entityStream = new AtlasEntityStream(existingEntity); -// entityStore.createOrUpdate(entityStream, false); -// LOG.info("Created bootstrap policies for connection {}", existingEntity.getAttribute(QUALIFIED_NAME)); -// } finally { -// RequestContext.get().setSkipAuthorizationCheck(false); -// } -// -// } } From 3e75d0f3af1894179ac7006d5048197e2f278487 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Tue, 23 Apr 2024 15:17:12 +0530 Subject: [PATCH 043/316] Fix order of attributes in contract and change dataContractVersion to integer --- .../contract/ContractPreProcessor.java | 18 ++++++----- .../contract/ContractVersionUtils.java | 6 ++-- .../preprocessor/contract/DataContract.java | 30 +++++++++++++++++-- 3 files changed, 40 insertions(+), 14 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index 173f0667538..5bf59414cb8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -33,9 +33,8 @@ public class ContractPreProcessor extends AbstractContractPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(ContractPreProcessor.class); public static final String ATTR_CONTRACT = "dataContractJson"; public static final String ATTR_VERSION = "dataContractVersion"; - public static final String REL_ATTR_GOVERNED_ASSET = "dataContractGovernedAsset"; - public static final String REL_ATTR_LATEST_CONTRACT = "dataContractLatest"; - public static final String REL_ATTR_GOVERNED_ASSET_CERTIFIED = "dataContractGovernedAssetCertified"; + public static final String REL_ATTR_GOVERNED_ASSET = "dataContractAsset"; + public static final String REL_ATTR_GOVERNED_ASSET_CERTIFIED = "dataContractAssetCertified"; public static final String REL_ATTR_PREVIOUS_VERSION = "dataContractPreviousVersion"; public static final String ASSET_ATTR_HAS_CONTRACT = "hasContract"; public static final String ASSET_ATTR_DESCRIPTION = "description"; @@ -113,11 +112,11 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con ContractVersionUtils versionUtil = new ContractVersionUtils(entity, context, entityRetriever, typeRegistry, entityStore, graph, discovery); AtlasEntity currentVersionEntity = versionUtil.getCurrentVersion(); - String latestVersion = "V1"; + int newVersionNumber = 1; if (currentVersionEntity != null) { // Contract already exist String qName = (String) currentVersionEntity.getAttribute(QUALIFIED_NAME); - Integer currentVersionNumber = Integer.valueOf(qName.substring(qName.lastIndexOf("/V") + 2)); + int currentVersionNumber = Integer.parseInt(qName.substring(qName.lastIndexOf("/V") + 2)); List attributes = getDiffAttributes(context, entity, currentVersionEntity); if (attributes.isEmpty()) { // No changes in the contract, Not creating new version @@ -129,7 +128,7 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con return; } else { // contract changed (metadata might/not changed). Create new version. - latestVersion = String.format("V%s", currentVersionNumber + 1); + newVersionNumber = currentVersionNumber + 1; // Attach previous version via rel entity.setRelationshipAttribute(REL_ATTR_PREVIOUS_VERSION, getAtlasObjectId(currentVersionEntity)); @@ -139,9 +138,12 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con } } - entity.setAttribute(QUALIFIED_NAME, String.format("%s/%s/%s", contractQName, VERSION_PREFIX, latestVersion)); - entity.setAttribute(ATTR_VERSION, latestVersion); + entity.setAttribute(QUALIFIED_NAME, String.format("%s/%s/V%s", contractQName, VERSION_PREFIX, newVersionNumber)); + entity.setAttribute(ATTR_VERSION, newVersionNumber); entity.setRelationshipAttribute(REL_ATTR_GOVERNED_ASSET, getAtlasObjectId(associatedAsset.getEntity())); + if (Objects.equals(entity.getAttribute(ATTR_CERTIFICATE_STATUS), DataContract.STATUS.VERIFIED.name()) ) { + entity.setRelationshipAttribute(REL_ATTR_GOVERNED_ASSET_CERTIFIED, getAtlasObjectId(associatedAsset.getEntity())); + } datasetAttributeSync(context, associatedAsset.getEntity(), contract, entity); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java index 3f93d305e91..e803e51b1f9 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java @@ -70,15 +70,15 @@ public AtlasEntity getCurrentVersion() throws AtlasBaseException { if (this.versionList == null) { extractAllVersions(); } + if (this.versionList == null) { + return null; + } Collections.sort(this.versionList, (e1, e2) -> { String e1QName = (String) e1.getAttribute(QUALIFIED_NAME); String e2QName = (String) e2.getAttribute(QUALIFIED_NAME); return e2QName.compareTo(e1QName); }); - if (this.versionList.isEmpty()) { - return null; - } return new AtlasEntity(this.versionList.get(0)); } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java index 18413f06cd8..d96c885c4fc 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java @@ -21,7 +21,8 @@ @JsonIgnoreProperties(ignoreUnknown = true) -@JsonPropertyOrder({"kind", "status", "template_version", "dataset", "type", "columns"}) +@JsonPropertyOrder({"kind", "status", "template_version", "datasource", "dataset", "type", "description", "owners", + "tags", "certificate", "columns"}) public class DataContract { @Valid @NotNull public String kind; @@ -31,12 +32,18 @@ public class DataContract { @JsonProperty(value = "template_version", defaultValue = "0.0.1") public String templateVersion; @Valid @NotNull + public String datasource; + @Valid @NotNull public String dataset; @Valid @NotNull public DATASET_TYPE type; + public String description; + public List owners; + public List tags; + public String certificate; @Valid - public List columns; - private Map unknownFields = new HashMap<>(); + public List columns; + private Map unknownFields = new HashMap<>(); @JsonSetter("type") public void setType(String type) throws AtlasBaseException { @@ -131,6 +138,23 @@ private boolean isSemVer(String version) { return matcher.matches(); } + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonPropertyOrder({"name"}) + public static final class BusinessTag { + @NotNull + public String name; + private Map unknownFields = new HashMap<>(); + + @JsonAnySetter + public void setUnknownFields(String key, Object value) { + unknownFields.put(key, value); + } + @JsonAnyGetter + public Map getUnknownFields() { + return unknownFields; + } + + } @JsonIgnoreProperties(ignoreUnknown = true) @JsonPropertyOrder({"name", "description", "data_type"}) public static final class Column { From 2e30f63bca2b291e144ee702c0d734e4aee60664 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Tue, 23 Apr 2024 17:45:58 +0530 Subject: [PATCH 044/316] Add bootstrap policy for DataContract CRU --- .../policies/bootstrap_entity_policies.json | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/addons/policies/bootstrap_entity_policies.json b/addons/policies/bootstrap_entity_policies.json index a889e0ef34d..ac8629faeb9 100644 --- a/addons/policies/bootstrap_entity_policies.json +++ b/addons/policies/bootstrap_entity_policies.json @@ -3004,6 +3004,108 @@ "entity-delete" ] } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "CREATE_DATA_CONTRACT", + "qualifiedName": "CREATE_DATA_CONTRACT", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:DataContract", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-create" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "READ_DATA_CONTRACT", + "qualifiedName": "READ_DATA_CONTRACT", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:DataContract", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-read" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "UPDATE_DATA_CONTRACT", + "qualifiedName": "UPDATE_DATA_CONTRACT", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:DataContract", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-update" + ] + } } ] } From 32104e90d347c416902c22a9450150fc8ccec2f6 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Wed, 24 Apr 2024 10:33:07 +0530 Subject: [PATCH 045/316] Fix qualifiedName of the contract --- .../org/apache/atlas/repository/Constants.java | 2 +- .../contract/AbstractContractPreProcessor.java | 3 +-- .../contract/ContractPreProcessor.java | 10 ++++++---- .../contract/ContractVersionUtils.java | 14 +++++++------- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 925d0a64769..2152921981c 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -400,7 +400,7 @@ public enum SupportedFileExtensions { XLSX, XLS, CSV } public static final String ATTR_ASSET_STARRED_BY = "assetStarredBy"; public static final String ATTR_ASSET_STARRED_AT = "assetStarredAt"; public static final String ATTR_CERTIFICATE_STATUS = "certificateStatus"; - + public static final String ATTR_CONTRACT = "dataContractJson"; public static final String STRUCT_STARRED_DETAILS = "StarredDetails"; public static final String KEYCLOAK_ROLE_ADMIN = "$admin"; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java index 4ef8d0ee6df..9b063d38f5a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java @@ -70,8 +70,7 @@ private void verifyAccess(AtlasEntityHeader entityHeader, AtlasPrivilege privile AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, privilege, entityHeader), errorMessage); } - AtlasEntity.AtlasEntityWithExtInfo getAssociatedAsset(String contractQName, String typeName) throws AtlasBaseException { - String datasetQName = contractQName.substring(0, contractQName.lastIndexOf('/')); + AtlasEntity.AtlasEntityWithExtInfo getAssociatedAsset(String datasetQName, String typeName) throws AtlasBaseException { Map uniqAttributes = new HashMap<>(); uniqAttributes.put(QUALIFIED_NAME, datasetQName); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index 5bf59414cb8..b1c2474d418 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -27,11 +27,11 @@ import static org.apache.atlas.AtlasErrorCode.*; import static org.apache.atlas.repository.Constants.ATTR_CERTIFICATE_STATUS; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; +import static org.apache.atlas.repository.Constants.ATTR_CONTRACT; import static org.apache.atlas.type.AtlasTypeUtil.getAtlasObjectId; public class ContractPreProcessor extends AbstractContractPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(ContractPreProcessor.class); - public static final String ATTR_CONTRACT = "dataContractJson"; public static final String ATTR_VERSION = "dataContractVersion"; public static final String REL_ATTR_GOVERNED_ASSET = "dataContractAsset"; public static final String REL_ATTR_GOVERNED_ASSET_CERTIFIED = "dataContractAssetCertified"; @@ -102,7 +102,9 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con String contractString = (String) entity.getAttribute(ATTR_CONTRACT); DataContract contract = DataContract.deserialize(contractString); - AtlasEntityWithExtInfo associatedAsset = getAssociatedAsset(contractQName, contract.type.name()); + String datasetQName = contractQName.substring(0, contractQName.lastIndexOf('/')); + contractQName = String.format("%s/%s/%s", datasetQName, contract.type.name(), CONTRACT_QUALIFIED_NAME_SUFFIX); + AtlasEntityWithExtInfo associatedAsset = getAssociatedAsset(datasetQName, contract.type.name()); authorizeContractCreateOrUpdate(entity, associatedAsset); @@ -110,7 +112,7 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con contractString = DataContract.serialize(contract); entity.setAttribute(ATTR_CONTRACT, contractString); - ContractVersionUtils versionUtil = new ContractVersionUtils(entity, context, entityRetriever, typeRegistry, entityStore, graph, discovery); + ContractVersionUtils versionUtil = new ContractVersionUtils(contractQName, context, entityRetriever, typeRegistry, entityStore, graph, discovery); AtlasEntity currentVersionEntity = versionUtil.getCurrentVersion(); int newVersionNumber = 1; if (currentVersionEntity != null) { @@ -138,7 +140,7 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con } } - entity.setAttribute(QUALIFIED_NAME, String.format("%s/%s/V%s", contractQName, VERSION_PREFIX, newVersionNumber)); + entity.setAttribute(QUALIFIED_NAME, String.format("%s/V%s", contractQName, newVersionNumber)); entity.setAttribute(ATTR_VERSION, newVersionNumber); entity.setRelationshipAttribute(REL_ATTR_GOVERNED_ASSET, getAtlasObjectId(associatedAsset.getEntity())); if (Objects.equals(entity.getAttribute(ATTR_CERTIFICATE_STATUS), DataContract.STATUS.VERIFIED.name()) ) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java index e803e51b1f9..149ec2b6ea7 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java @@ -23,14 +23,13 @@ public class ContractVersionUtils { public final EntityGraphRetriever entityRetriever; private final AtlasTypeRegistry atlasTypeRegistry; private AtlasEntityStore entityStore; - - private AtlasEntity entity; + private String contractQName; public final AtlasGraph graph; private List versionList; private EntityDiscoveryService discovery; - public ContractVersionUtils(AtlasEntity entity, EntityMutationContext context, EntityGraphRetriever entityRetriever, + public ContractVersionUtils(String contractQName, EntityMutationContext context, EntityGraphRetriever entityRetriever, AtlasTypeRegistry atlasTypeRegistry, AtlasEntityStore entityStore, AtlasGraph graph, EntityDiscoveryService discovery) { this.context = context; @@ -38,13 +37,11 @@ public ContractVersionUtils(AtlasEntity entity, EntityMutationContext context, E this.atlasTypeRegistry = atlasTypeRegistry; this.graph = graph; this.entityStore = entityStore; - this.entity = entity; + this.contractQName = contractQName; this.discovery = discovery; } private void extractAllVersions() throws AtlasBaseException { - String contractQName = (String) entity.getAttribute(QUALIFIED_NAME); - String datasetQName = contractQName.substring(0, contractQName.lastIndexOf("/contract")); List ret = new ArrayList<>(); IndexSearchParams indexSearchParams = new IndexSearchParams(); @@ -52,11 +49,14 @@ private void extractAllVersions() throws AtlasBaseException { List mustClauseList = new ArrayList(); mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", CONTRACT_ENTITY_TYPE))); - mustClauseList.add(mapOf("wildcard", mapOf(QUALIFIED_NAME, String.format("%s/contract/version/*", datasetQName)))); + mustClauseList.add(mapOf("wildcard", mapOf(QUALIFIED_NAME, String.format("%s/*", contractQName)))); dsl.put("query", mapOf("bool", mapOf("must", mustClauseList))); + Set attributes = new HashSet<>(); + attributes.add(ATTR_CONTRACT); indexSearchParams.setDsl(dsl); + indexSearchParams.setAttributes(attributes); indexSearchParams.setSuppressLogs(true); AtlasSearchResult result = discovery.directIndexSearch(indexSearchParams); From 8185bfe0ad29e6b6831159220f56cb1d7a50a7a1 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Wed, 24 Apr 2024 12:33:11 +0530 Subject: [PATCH 046/316] Add asset guid in DataContract asset and minor fixes --- .../v2/preprocessor/contract/ContractPreProcessor.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index b1c2474d418..e86c38514b9 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -33,7 +33,8 @@ public class ContractPreProcessor extends AbstractContractPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(ContractPreProcessor.class); public static final String ATTR_VERSION = "dataContractVersion"; - public static final String REL_ATTR_GOVERNED_ASSET = "dataContractAsset"; + public static final String ATTR_ASSET_GUID = "dataContractAssetGuid"; + public static final String REL_ATTR_GOVERNED_ASSET = "dataContractAssetLatest"; public static final String REL_ATTR_GOVERNED_ASSET_CERTIFIED = "dataContractAssetCertified"; public static final String REL_ATTR_PREVIOUS_VERSION = "dataContractPreviousVersion"; public static final String ASSET_ATTR_HAS_CONTRACT = "hasContract"; @@ -127,7 +128,7 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con } else if (isEqualContract(contractString, (String) currentVersionEntity.getAttribute(ATTR_CONTRACT))) { // No change in contract, metadata changed updateExistingVersion(context, entity, currentVersionEntity); - return; + newVersionNumber = currentVersionNumber; } else { // contract changed (metadata might/not changed). Create new version. newVersionNumber = currentVersionNumber + 1; @@ -142,6 +143,7 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con } entity.setAttribute(QUALIFIED_NAME, String.format("%s/V%s", contractQName, newVersionNumber)); entity.setAttribute(ATTR_VERSION, newVersionNumber); + entity.setAttribute(ATTR_ASSET_GUID, associatedAsset.getEntity().getGuid()); entity.setRelationshipAttribute(REL_ATTR_GOVERNED_ASSET, getAtlasObjectId(associatedAsset.getEntity())); if (Objects.equals(entity.getAttribute(ATTR_CERTIFICATE_STATUS), DataContract.STATUS.VERIFIED.name()) ) { entity.setRelationshipAttribute(REL_ATTR_GOVERNED_ASSET_CERTIFIED, getAtlasObjectId(associatedAsset.getEntity())); From e863c1369545ad2a0bda17bc3e3b25e93100dc06 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Wed, 24 Apr 2024 14:13:52 +0530 Subject: [PATCH 047/316] Inconsistency in relationship fixed for data contract<>asset --- .../contract/ContractPreProcessor.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index e86c38514b9..17ee1223018 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -34,8 +34,8 @@ public class ContractPreProcessor extends AbstractContractPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(ContractPreProcessor.class); public static final String ATTR_VERSION = "dataContractVersion"; public static final String ATTR_ASSET_GUID = "dataContractAssetGuid"; - public static final String REL_ATTR_GOVERNED_ASSET = "dataContractAssetLatest"; - public static final String REL_ATTR_GOVERNED_ASSET_CERTIFIED = "dataContractAssetCertified"; + public static final String REL_ATTR_LATEST_CONTRACT = "dataContractLatest"; + public static final String REL_ATTR_GOVERNED_ASSET_CERTIFIED = "dataContractLatestCertified"; public static final String REL_ATTR_PREVIOUS_VERSION = "dataContractPreviousVersion"; public static final String ASSET_ATTR_HAS_CONTRACT = "hasContract"; public static final String ASSET_ATTR_DESCRIPTION = "description"; @@ -144,10 +144,6 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con entity.setAttribute(QUALIFIED_NAME, String.format("%s/V%s", contractQName, newVersionNumber)); entity.setAttribute(ATTR_VERSION, newVersionNumber); entity.setAttribute(ATTR_ASSET_GUID, associatedAsset.getEntity().getGuid()); - entity.setRelationshipAttribute(REL_ATTR_GOVERNED_ASSET, getAtlasObjectId(associatedAsset.getEntity())); - if (Objects.equals(entity.getAttribute(ATTR_CERTIFICATE_STATUS), DataContract.STATUS.VERIFIED.name()) ) { - entity.setRelationshipAttribute(REL_ATTR_GOVERNED_ASSET_CERTIFIED, getAtlasObjectId(associatedAsset.getEntity())); - } datasetAttributeSync(context, associatedAsset.getEntity(), contract, entity); @@ -244,6 +240,13 @@ private void datasetAttributeSync(EntityMutationContext context, AtlasEntity ass if (associatedAsset.getAttribute(ASSET_ATTR_HAS_CONTRACT) == null || associatedAsset.getAttribute(ASSET_ATTR_HAS_CONTRACT).equals(false)) { entity.setAttribute(ASSET_ATTR_HAS_CONTRACT, true); } + + // Update relationship with contract + entity.setRelationshipAttribute(REL_ATTR_LATEST_CONTRACT, getAtlasObjectId(contractAsset)); + if (Objects.equals(contractAsset.getAttribute(ATTR_CERTIFICATE_STATUS), DataContract.STATUS.VERIFIED.name()) ) { + entity.setRelationshipAttribute(REL_ATTR_GOVERNED_ASSET_CERTIFIED, getAtlasObjectId(contractAsset)); + } + AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(entity.getGuid()); AtlasEntityType entityType = ensureEntityType(entity.getTypeName()); AtlasEntityComparator entityComparator = new AtlasEntityComparator(typeRegistry, entityRetriever, context.getGuidAssignments(), true, true); From 297152040620b62e366dda96f0e5d48b618edafd Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Wed, 24 Apr 2024 17:04:09 +0530 Subject: [PATCH 048/316] Removed maven release for data-contract branch --- .github/workflows/maven.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 39703359df6..0977cb36a25 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,7 +26,6 @@ on: - development - master - lineageondemand - - data-contract jobs: build: From 963071582f7ae1e98ae23e3ca740141e58ea5de8 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Wed, 24 Apr 2024 17:29:35 +0530 Subject: [PATCH 049/316] change in policyUpdate --- .../datamesh/AbstractDomainPreProcessor.java | 20 +++++++++++-------- .../datamesh/DataProductPreProcessor.java | 6 +++--- .../datamesh/DomainPreProcessor.java | 4 ++-- 3 files changed, 17 insertions(+), 13 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 985ecb786d5..32629018c44 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -143,18 +143,22 @@ protected void updatePolicy(String currentQualifiedName, String updatedQualified if (CollectionUtils.isNotEmpty(policies)) { AtlasEntityType entityType = typeRegistry.getEntityTypeByName(POLICY_ENTITY_TYPE); for (AtlasEntityHeader policy : policies) { - LOG.info("Updating policy {} for entity {}", policy.getGuid(), currentQualifiedName); - AtlasEntity policyEntity = entityRetriever.toAtlasEntity(policy.getGuid()); - List policyResources = (List) policyEntity.getAttribute(ATTR_POLICY_RESOURCES); - LOG.info("Policy resources {}", policyResources); + AtlasVertex policyVertex = entityRetriever.getEntityVertex(policy.getGuid()); + + AtlasEntity policyEntity = entityRetriever.toAtlasEntity(policyVertex); + LOG.info("Policy entity guid {} and policy header guid {}", policyEntity.getGuid(), policy.getGuid()); + + List policyResources = (List) policyVertex.getProperty(ATTR_POLICY_RESOURCES, List.class); policyResources.remove(currentResource); policyResources.add(updatedResource); - AtlasVertex policyVertex = entityRetriever.getEntityVertex(policy.getGuid()); - LOG.info("Policy Vertex {}", policyVertex); - LOG.info("Context {}", context); + LOG.info("Policy resources after update {}", policyResources); + + policyVertex.removeProperty(ATTR_POLICY_RESOURCES); policyEntity.setAttribute(ATTR_POLICY_RESOURCES, policyResources); + context.addUpdated(policyEntity.getGuid(), policyEntity, entityType, policyVertex); + LOG.info("Context {}", context); } } @@ -167,7 +171,7 @@ protected void updatePolicy(String currentQualifiedName, String updatedQualified protected List getPolicy(String resource) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("getPolicy"); try { - List mustClauseList = new ArrayList(); + List> mustClauseList = new ArrayList<>(); mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", POLICY_ENTITY_TYPE))); mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); mustClauseList.add(mapOf("terms", mapOf("policyResources", Arrays.asList(resource)))); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 14b0b6d0fab..6d094e1ed25 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -25,7 +25,7 @@ import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; public class DataProductPreProcessor extends AbstractDomainPreProcessor { - private static final Logger LOG = LoggerFactory.getLogger(DomainPreProcessor.class); + private static final Logger LOG = LoggerFactory.getLogger(DataProductPreProcessor.class); private AtlasEntityHeader parentDomain; private EntityMutationContext context; public DataProductPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, @@ -209,7 +209,7 @@ private void productExists(String productName, String parentDomainQualifiedName) boolean exists = false; try { - List mustClauseList = new ArrayList(); + List> mustClauseList = new ArrayList(); mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", DATA_PRODUCT_ENTITY_TYPE))); mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); mustClauseList.add(mapOf("term", mapOf("name.keyword", productName))); @@ -219,7 +219,7 @@ private void productExists(String productName, String parentDomainQualifiedName) if (parentDomain != null) { mustClauseList.add(mapOf("term", mapOf("parentDomainQualifiedName", parentDomainQualifiedName))); } else { - List mustNotClauseList = new ArrayList(); + List> mustNotClauseList = new ArrayList(); mustNotClauseList.add(mapOf("exists", mapOf("field", "parentDomainQualifiedName"))); bool.put("must_not", mustNotClauseList); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index abe27701e03..ee6a1a76c20 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -319,7 +319,7 @@ private void domainExists(String domainName, String parentDomainQualifiedName) t boolean exists = false; try { - List mustClauseList = new ArrayList(); + List> mustClauseList = new ArrayList(); mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", DATA_DOMAIN_ENTITY_TYPE))); mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); mustClauseList.add(mapOf("term", mapOf("name.keyword", domainName))); @@ -329,7 +329,7 @@ private void domainExists(String domainName, String parentDomainQualifiedName) t if (parentDomain != null) { mustClauseList.add(mapOf("term", mapOf("parentDomainQualifiedName", parentDomainQualifiedName))); } else { - List mustNotClauseList = new ArrayList(); + List> mustNotClauseList = new ArrayList(); mustNotClauseList.add(mapOf("exists", mapOf("field", "parentDomainQualifiedName"))); bool.put("must_not", mustNotClauseList); } From 43562a9d68df9ae06ba4749aa2ac4fac3fb7ccf6 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Wed, 24 Apr 2024 17:42:02 +0530 Subject: [PATCH 050/316] check for parentDomain for products corrected --- .../graph/v2/preprocessor/datamesh/DataProductPreProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 6d094e1ed25..6692845765a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -72,7 +72,7 @@ private void processCreateProduct(AtlasEntity entity, AtlasVertex vertex) throws } private static String createQualifiedName(String parentDomainQualifiedName) throws AtlasBaseException { - if (StringUtils.isNotEmpty(parentDomainQualifiedName)) { + if (StringUtils.isEmpty(parentDomainQualifiedName)) { throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Parent Domain Qualified Name cannot be empty or null"); } return parentDomainQualifiedName + "/product/" + getUUID(); From 8a16887d477fbc715730f5c2fad3bd93bbfac580 Mon Sep 17 00:00:00 2001 From: Rajat Movaliya Date: Wed, 24 Apr 2024 20:25:24 +0530 Subject: [PATCH 051/316] fix: lineage api node limit exceeding --- .../java/org/apache/atlas/discovery/EntityLineageService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index 6927b1d1c4f..7c69099e746 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -653,7 +653,7 @@ private List getFilteredAtlasEdges(AtlasVertex outVertex, String proc } private boolean isEntityTraversalLimitReached(AtomicInteger entitiesTraversed) { - return entitiesTraversed.get() == getLineageMaxNodeAllowedCount(); + return entitiesTraversed.get() >= getLineageMaxNodeAllowedCount(); } @Override From 86349970dc1be6b45163f5e166825a2e97b68ab1 Mon Sep 17 00:00:00 2001 From: nishantmunjal7 <122665017+nishantmunjal7@users.noreply.github.com> Date: Thu, 25 Apr 2024 09:47:13 +0530 Subject: [PATCH 052/316] Update bootstrap_heka_policies.json --- addons/policies/bootstrap_heka_policies.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/addons/policies/bootstrap_heka_policies.json b/addons/policies/bootstrap_heka_policies.json index d092186d789..4a9aeb39356 100644 --- a/addons/policies/bootstrap_heka_policies.json +++ b/addons/policies/bootstrap_heka_policies.json @@ -13,6 +13,7 @@ "policyServiceName": "heka", "policyType": "deny", "policyPriority": 1, + "isPolicyEnabled": false, "policyUsers": [], "policyGroups": [], "policyRoles": @@ -32,4 +33,4 @@ } } ] -} \ No newline at end of file +} From a49807e4fcd4dbd4eb83b43b067d2c8f7f343bca Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Thu, 25 Apr 2024 10:51:10 +0530 Subject: [PATCH 053/316] issue in dataProduct qn update resolved --- pom.xml | 2 +- .../graph/v2/preprocessor/datamesh/DomainPreProcessor.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pom.xml b/pom.xml index 1cc9aa70dc9..3abc39041b7 100644 --- a/pom.xml +++ b/pom.xml @@ -717,7 +717,7 @@ 4.4.13 2.12.4 2.12.4 - 0.6.03 + 0.6.04 0.5.3 1 3.1.0 diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index ee6a1a76c20..318e5251795 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -265,7 +265,7 @@ private void moveChildDataProductToAnotherDomain(AtlasVertex productVertex, LOG.info("Moving dataProduct {} to Domain {}", productName, targetDomainQualifiedName); Map updatedAttributes = new HashMap<>(); - String currentQualifiedName = productVertex.getProperty(PARENT_DOMAIN_QN, String.class); + String currentQualifiedName = productVertex.getProperty(QUALIFIED_NAME, String.class); String updatedQualifiedName = currentQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); productVertex.setProperty(QUALIFIED_NAME, updatedQualifiedName); From ab04e1ffec9b2b6e6f228883c5e4ea945d96b73f Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Thu, 25 Apr 2024 10:56:10 +0530 Subject: [PATCH 054/316] removed version change --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3abc39041b7..1cc9aa70dc9 100644 --- a/pom.xml +++ b/pom.xml @@ -717,7 +717,7 @@ 4.4.13 2.12.4 2.12.4 - 0.6.04 + 0.6.03 0.5.3 1 3.1.0 From b63c3164723b3bb5d4c769a3c6c12654b460bf21 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Thu, 25 Apr 2024 12:01:12 +0530 Subject: [PATCH 055/316] feat: implement redis as our cache provider --- .../service/redis/AbstractRedisService.java | 28 ++++++++- .../service/redis/NoRedisServiceImpl.java | 15 +++++ .../atlas/service/redis/RedisService.java | 6 ++ .../service/redis/RedisServiceLocalImpl.java | 15 +++++ .../janus/AtlasElasticsearchQuery.java | 27 +++++++-- .../graphdb/janus/SearchContextCache.java | 57 +++++++++++-------- 6 files changed, 118 insertions(+), 30 deletions(-) diff --git a/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java b/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java index 0ae20a60cc4..49eb23b661e 100644 --- a/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java +++ b/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java @@ -71,6 +71,25 @@ public void releaseDistributedLock(String key) { } } + @Override + public String getValue(String key) { + // If value doesn't exist, return null else return the value + return (String) redisClient.getBucket(convertToNamespace(key)).get(); + } + + @Override + public String putValue(String key, String value) { + // Put the value in the redis cache with TTL + redisClient.getBucket(convertToNamespace(key)).set(value, 30, TimeUnit.SECONDS); + return value; + } + + @Override + public void removeValue(String key) { + // Remove the value from the redis cache + redisClient.getBucket(convertToNamespace(key)).delete(); + } + private String getHostAddress() throws UnknownHostException { return InetAddress.getLocalHost().getHostAddress(); } @@ -85,6 +104,11 @@ private Config initAtlasConfig() throws AtlasException { return redisConfig; } + private String convertToNamespace(String key){ + // Append key with namespace :atlas + return "atlas:"+key; + } + Config getLocalConfig() throws AtlasException { Config config = initAtlasConfig(); config.useSingleServer() @@ -94,11 +118,11 @@ Config getLocalConfig() throws AtlasException { return config; } - Config getProdConfig() throws AtlasException { + protected Config getProdConfig() throws AtlasException { Config config = initAtlasConfig(); config.useSentinelServers() .setClientName(ATLAS_METASTORE_SERVICE) - .setReadMode(ReadMode.MASTER_SLAVE) + .setReadMode(ReadMode.MASTER) .setCheckSentinelsList(false) .setKeepAlive(true) .setMasterConnectionMinimumIdleSize(10) diff --git a/common/src/main/java/org/apache/atlas/service/redis/NoRedisServiceImpl.java b/common/src/main/java/org/apache/atlas/service/redis/NoRedisServiceImpl.java index 96a8fadc99f..5bc089d9557 100644 --- a/common/src/main/java/org/apache/atlas/service/redis/NoRedisServiceImpl.java +++ b/common/src/main/java/org/apache/atlas/service/redis/NoRedisServiceImpl.java @@ -29,6 +29,21 @@ public void releaseDistributedLock(String key) { //do nothing } + @Override + public String getValue(String key) { + return null; + } + + @Override + public String putValue(String key, String value) { + return null; + } + + @Override + public void removeValue(String key) { + + } + @Override public Logger getLogger() { return LOG; diff --git a/common/src/main/java/org/apache/atlas/service/redis/RedisService.java b/common/src/main/java/org/apache/atlas/service/redis/RedisService.java index 1475f93e832..6cb2f04ada3 100644 --- a/common/src/main/java/org/apache/atlas/service/redis/RedisService.java +++ b/common/src/main/java/org/apache/atlas/service/redis/RedisService.java @@ -8,6 +8,12 @@ public interface RedisService { void releaseDistributedLock(String key); + String getValue(String key); + + String putValue(String key, String value); + + void removeValue(String key); + Logger getLogger(); } diff --git a/common/src/main/java/org/apache/atlas/service/redis/RedisServiceLocalImpl.java b/common/src/main/java/org/apache/atlas/service/redis/RedisServiceLocalImpl.java index 2eb774920ef..1ddad0f3e80 100644 --- a/common/src/main/java/org/apache/atlas/service/redis/RedisServiceLocalImpl.java +++ b/common/src/main/java/org/apache/atlas/service/redis/RedisServiceLocalImpl.java @@ -21,6 +21,21 @@ public void init() throws AtlasException { LOG.info("Local redis client created successfully."); } + @Override + public String getValue(String key) { + return null; + } + + @Override + public String putValue(String key, String value) { + return null; + } + + @Override + public void removeValue(String key) { + + } + @Override public Logger getLogger() { return LOG; diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java index 9d2e2489e8a..a4e6bde67b6 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java @@ -172,10 +172,15 @@ private Map runQueryWithLowLevelClient(String query) throws Atla private DirectIndexQueryResult performAsyncDirectIndexQuery(SearchParams searchParams) throws AtlasBaseException, IOException { DirectIndexQueryResult result = null; + boolean contextIdExists = StringUtils.isNotEmpty(searchParams.getSearchContextId()) && searchParams.getSearchContextSequenceNo() != null; try { - if(StringUtils.isNotEmpty(searchParams.getSearchContextId()) && searchParams.getSearchContextSequenceNo() != null) { + if(contextIdExists) { // If the search context id and greater sequence no is present, then we need to delete the previous search context async - processRequestWithSameSearchContextId(searchParams); + try { + processRequestWithSameSearchContextId(searchParams); + } catch (Exception e) { + LOG.error("Failed to process the request with the same search context ID {}", e.getMessage()); + } } AsyncQueryResult response = submitAsyncSearch(searchParams, false).get(); @@ -189,8 +194,12 @@ private DirectIndexQueryResult performAsyncDirectIndexQuery(SearchParams searchP String esSearchId = response.getId(); String searchContextId = searchParams.getSearchContextId(); Integer searchContextSequenceNo = searchParams.getSearchContextSequenceNo(); - if (StringUtils.isNotEmpty(searchContextId) && searchContextSequenceNo != null) { - SearchContextCache.put(searchContextId, searchContextSequenceNo, esSearchId); + if (contextIdExists) { + try { + SearchContextCache.put(searchContextId, searchContextSequenceNo, esSearchId); + } catch (Exception e) { + LOG.error("Failed to update the search context cache {}", e.getMessage()); + } } response = getAsyncSearchResponse(searchParams, esSearchId).get(); if (response == null) { @@ -204,6 +213,15 @@ private DirectIndexQueryResult performAsyncDirectIndexQuery(SearchParams searchP }catch (Exception e) { LOG.error("Failed to execute direct query on ES {}", e.getMessage()); throw new AtlasBaseException(AtlasErrorCode.INDEX_SEARCH_FAILED, e.getMessage()); + } finally { + if (contextIdExists) { + // If the search context id is present, then we need to remove the search context from the cache + try { + SearchContextCache.remove(searchParams.getSearchContextId()); + } catch (Exception e) { + LOG.error("Failed to remove the search context from the cache {}", e.getMessage()); + } + } } return result; } @@ -228,7 +246,6 @@ private void processRequestWithSameSearchContextId(SearchParams searchParams) { LOG.debug("Deleting the previous async search response with ID {}", previousESSearchId); // If the search ID exists, then we need to delete the search context deleteAsyncSearchResponse(previousESSearchId); - SearchContextCache.remove(currentSearchContextId); } } diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/SearchContextCache.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/SearchContextCache.java index 1780e5d1f90..6e1559010e3 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/SearchContextCache.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/SearchContextCache.java @@ -2,48 +2,59 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; +import org.apache.atlas.service.redis.AbstractRedisService; +import org.apache.atlas.service.redis.RedisService; +import org.slf4j.Logger; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Component; import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; +@Component public class SearchContextCache { - private static final Cache> searchContextCache = CacheBuilder.newBuilder() - .maximumSize(200) - .expireAfterWrite(30, TimeUnit.SECONDS) - .build(); + private static RedisService redisService = null; + + public static final String INVALID_SEQUENCE = "invalid_sequence"; + + + public SearchContextCache(@Qualifier("redisServiceImpl") RedisService redisService) { + SearchContextCache.redisService = redisService; + } + public static void put(String key, Integer sequence, String esAsyncId) { - HashMap entry = new HashMap<>(); - entry.put(sequence, esAsyncId); - searchContextCache.put(key, entry); + // Build the string in format `sequence/esAsyncId` and store it in redis + String val = sequence + "/" + esAsyncId; + redisService.putValue(key, val); } - public static HashMap get(String key){ - return searchContextCache.getIfPresent(key); + public static String get(String key){ + return redisService.getValue(key); } public static String getESAsyncSearchIdFromContextCache(String key, Integer sequence){ //Get the context cache for the given key - HashMap contextCache = get(key); + String contextCache = get(key); if(contextCache == null || sequence == null){ return null; } - //Find the highest sequence number - int maxStoredSequence = 0; - for (Integer seq : contextCache.keySet()) { - if (seq > maxStoredSequence) { - maxStoredSequence = seq; - } + // Split the context cache to get the sequence and ESAsyncId + String[] contextCacheSplit = contextCache.split("/"); + if(contextCacheSplit.length != 2){ + return null; + } + int seq = Integer.parseInt(contextCacheSplit[0]); + if(sequence > seq){ + return contextCacheSplit[1]; + } else if (sequence < seq) { + return INVALID_SEQUENCE; } - //If the given sequence is greater than the max stored sequence, return the ESAsyncId else return null - return sequence > maxStoredSequence ? contextCache.getOrDefault(maxStoredSequence, null) : null; + return null; } public static void remove(String key) { - searchContextCache.invalidate(key); - } - - public static void clear() { - searchContextCache.cleanUp(); + redisService.removeValue(key); } } + From 203b45f46301420c831478949c6530bfc9a95efc Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Thu, 25 Apr 2024 12:11:29 +0530 Subject: [PATCH 056/316] fix: just tap the utm tags coming from product --- .../src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java index 70c1f57cc3a..144e1171d1e 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java @@ -94,6 +94,7 @@ public class DiscoveryREST { private static final String INDEXSEARCH_TAG_NAME = "indexsearch"; private static final Set TRACKING_UTM_TAGS = new HashSet<>(Arrays.asList("ui_main_list", "ui_popup_searchbar")); + private static final String UTM_TAG_FROM_PRODUCT = "project_webapp"; @Inject public DiscoveryREST(AtlasTypeRegistry typeRegistry, AtlasDiscoveryService discoveryService, @@ -433,7 +434,7 @@ public AtlasSearchResult indexSearch(@Context HttpServletRequest servletRequest, } throw abe; } finally { - if(CollectionUtils.isNotEmpty(parameters.getUtmTags())) { + if(CollectionUtils.isNotEmpty(parameters.getUtmTags()) && parameters.getUtmTags().contains(UTM_TAG_FROM_PRODUCT)) { AtlasPerfMetrics.Metric indexsearchMetric = new AtlasPerfMetrics.Metric(INDEXSEARCH_TAG_NAME); indexsearchMetric.addTag("utmTag", "other"); for (String utmTag : parameters.getUtmTags()) { From b7e3f15b18beb0481fe00c1af2fe7cf2e447957a Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Thu, 25 Apr 2024 13:10:08 +0530 Subject: [PATCH 057/316] fix: make change to add sourse as tag --- .../main/java/org/apache/atlas/web/rest/DiscoveryREST.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java index 144e1171d1e..590e3cb0bfa 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/DiscoveryREST.java @@ -434,15 +434,19 @@ public AtlasSearchResult indexSearch(@Context HttpServletRequest servletRequest, } throw abe; } finally { - if(CollectionUtils.isNotEmpty(parameters.getUtmTags()) && parameters.getUtmTags().contains(UTM_TAG_FROM_PRODUCT)) { + if(CollectionUtils.isNotEmpty(parameters.getUtmTags())) { AtlasPerfMetrics.Metric indexsearchMetric = new AtlasPerfMetrics.Metric(INDEXSEARCH_TAG_NAME); indexsearchMetric.addTag("utmTag", "other"); + indexsearchMetric.addTag("source", "other"); for (String utmTag : parameters.getUtmTags()) { if (TRACKING_UTM_TAGS.contains(utmTag)) { indexsearchMetric.addTag("utmTag", utmTag); break; } } + if (parameters.getUtmTags().contains(UTM_TAG_FROM_PRODUCT)) { + indexsearchMetric.addTag("source", UTM_TAG_FROM_PRODUCT); + } indexsearchMetric.addTag("name", INDEXSEARCH_TAG_NAME); indexsearchMetric.setTotalTimeMSecs(System.currentTimeMillis() - startTime); RequestContext.get().addApplicationMetrics(indexsearchMetric); From 15068743fcb49421b045d5e49cd676765060c2cc Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Thu, 25 Apr 2024 14:10:36 +0530 Subject: [PATCH 058/316] getting policyResource from entity --- .../v2/preprocessor/datamesh/AbstractDomainPreProcessor.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 32629018c44..92dd73ccb96 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -148,12 +148,11 @@ protected void updatePolicy(String currentQualifiedName, String updatedQualified AtlasEntity policyEntity = entityRetriever.toAtlasEntity(policyVertex); LOG.info("Policy entity guid {} and policy header guid {}", policyEntity.getGuid(), policy.getGuid()); - List policyResources = (List) policyVertex.getProperty(ATTR_POLICY_RESOURCES, List.class); + List policyResources = (List) policyEntity.getAttribute(ATTR_POLICY_RESOURCES); policyResources.remove(currentResource); policyResources.add(updatedResource); LOG.info("Policy resources after update {}", policyResources); - policyVertex.removeProperty(ATTR_POLICY_RESOURCES); policyEntity.setAttribute(ATTR_POLICY_RESOURCES, policyResources); From 0af6d84d3dc438fe4c08e042b754f8db10d3aaff Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Thu, 25 Apr 2024 15:55:39 +0530 Subject: [PATCH 059/316] DG-1357 Custom metadata access for mesh assets --- .../policy_cache_transformer_persona.json | 42 +++++++++++++++++-- 1 file changed, 39 insertions(+), 3 deletions(-) diff --git a/addons/static/templates/policy_cache_transformer_persona.json b/addons/static/templates/policy_cache_transformer_persona.json index a61b671090b..17bc75c10ee 100644 --- a/addons/static/templates/policy_cache_transformer_persona.json +++ b/addons/static/templates/policy_cache_transformer_persona.json @@ -480,7 +480,19 @@ "actions": ["add-relationship", "update-relationship", "remove-relationship"] } ], - + "persona-domain-business-update-metadata": [ + { + "policyType": "ACCESS", + "policyResourceCategory": "ENTITY", + "resources": [ + "entity:{entity}", + "entity-type:DataDomain", + "entity-classification:*", + "entity-business-metadata:*" + ], + "actions": ["entity-update-business-metadata"] + } + ], "persona-domain-sub-domain-read": [ { @@ -589,6 +601,19 @@ "actions": ["entity-delete"] } ], + "persona-domain-sub-domain-business-update-metadata": [ + { + "policyType": "ACCESS", + "policyResourceCategory": "ENTITY", + "resources": [ + "entity:{entity}/*domain/*", + "entity-type:DataDomain", + "entity-classification:*", + "entity-business-metadata:*" + ], + "actions": ["entity-update-business-metadata"] + } + ], "persona-domain-product-read": [ { @@ -719,8 +744,19 @@ "actions": ["entity-delete"] } ], - - + "persona-domain-product-business-update-metadata": [ + { + "policyType": "ACCESS", + "policyResourceCategory": "ENTITY", + "resources": [ + "entity:{entity}/*product/*", + "entity-type:DataProduct", + "entity-classification:*", + "entity-business-metadata:*" + ], + "actions": ["entity-update-business-metadata"] + } + ], "select": [ { From b195ce50a41cc1ce9f6572bacacbd8dce3199020 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Thu, 25 Apr 2024 16:52:12 +0530 Subject: [PATCH 060/316] added logs to debug --- .../v2/preprocessor/datamesh/AbstractDomainPreProcessor.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 92dd73ccb96..71ea4cb8c45 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -149,7 +149,9 @@ protected void updatePolicy(String currentQualifiedName, String updatedQualified LOG.info("Policy entity guid {} and policy header guid {}", policyEntity.getGuid(), policy.getGuid()); List policyResources = (List) policyEntity.getAttribute(ATTR_POLICY_RESOURCES); + LOG.info("Policy resources before update {}", policyResources); policyResources.remove(currentResource); + LOG.info("Policy resources after removing oldQN {}", policyResources); policyResources.add(updatedResource); LOG.info("Policy resources after update {}", policyResources); From 997c7a629a6c9dec3f7cc1b511941ff6b46d7e2c Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Thu, 25 Apr 2024 16:53:09 +0530 Subject: [PATCH 061/316] removed noise from logs --- .../v2/preprocessor/datamesh/AbstractDomainPreProcessor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 71ea4cb8c45..0bbf94afe22 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -159,7 +159,6 @@ protected void updatePolicy(String currentQualifiedName, String updatedQualified policyEntity.setAttribute(ATTR_POLICY_RESOURCES, policyResources); context.addUpdated(policyEntity.getGuid(), policyEntity, entityType, policyVertex); - LOG.info("Context {}", context); } } From f7e9f3523d726901a4c6aad61cc3efcd49ff32b2 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Thu, 25 Apr 2024 17:15:16 +0530 Subject: [PATCH 062/316] fix: remove local change --- .../org/apache/atlas/service/redis/AbstractRedisService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java b/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java index 49eb23b661e..5bd2785ba04 100644 --- a/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java +++ b/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java @@ -122,7 +122,7 @@ protected Config getProdConfig() throws AtlasException { Config config = initAtlasConfig(); config.useSentinelServers() .setClientName(ATLAS_METASTORE_SERVICE) - .setReadMode(ReadMode.MASTER) + .setReadMode(ReadMode.MASTER_SLAVE) .setCheckSentinelsList(false) .setKeepAlive(true) .setMasterConnectionMinimumIdleSize(10) From 103d8ed349b29e659bd0094ae6b04d932b94adee Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Thu, 25 Apr 2024 18:01:13 +0530 Subject: [PATCH 063/316] handles policyResource not found case --- .../datamesh/AbstractDomainPreProcessor.java | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 0bbf94afe22..3755577332f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -29,6 +29,7 @@ import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; @@ -146,19 +147,22 @@ protected void updatePolicy(String currentQualifiedName, String updatedQualified AtlasVertex policyVertex = entityRetriever.getEntityVertex(policy.getGuid()); AtlasEntity policyEntity = entityRetriever.toAtlasEntity(policyVertex); - LOG.info("Policy entity guid {} and policy header guid {}", policyEntity.getGuid(), policy.getGuid()); List policyResources = (List) policyEntity.getAttribute(ATTR_POLICY_RESOURCES); - LOG.info("Policy resources before update {}", policyResources); - policyResources.remove(currentResource); - LOG.info("Policy resources after removing oldQN {}", policyResources); - policyResources.add(updatedResource); - LOG.info("Policy resources after update {}", policyResources); + // Check if currentResource exists in the list before removing it + if (policyResources.contains(currentResource)) { + policyResources.remove(currentResource); + policyResources.add(updatedResource); + LOG.info("Policy resources after updating QN: {}", policyResources); + } else { + LOG.info("CurrentResource {} not found in the policy resources. Skipping update.", currentResource); + } policyVertex.removeProperty(ATTR_POLICY_RESOURCES); - policyEntity.setAttribute(ATTR_POLICY_RESOURCES, policyResources); - - context.addUpdated(policyEntity.getGuid(), policyEntity, entityType, policyVertex); + if(AtlasGraphUtilsV2.getState(policyVertex) == AtlasEntity.Status.ACTIVE){ + policyEntity.setAttribute(ATTR_POLICY_RESOURCES, policyResources); + context.addUpdated(policyEntity.getGuid(), policyEntity, entityType, policyVertex); + } } } From 9235ce11619dc8c5f838e6abc6e38b90a4544a7d Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Thu, 25 Apr 2024 18:02:52 +0530 Subject: [PATCH 064/316] removing policyResources after check --- .../v2/preprocessor/datamesh/AbstractDomainPreProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 3755577332f..a6c4364708c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -158,8 +158,8 @@ protected void updatePolicy(String currentQualifiedName, String updatedQualified LOG.info("CurrentResource {} not found in the policy resources. Skipping update.", currentResource); } - policyVertex.removeProperty(ATTR_POLICY_RESOURCES); if(AtlasGraphUtilsV2.getState(policyVertex) == AtlasEntity.Status.ACTIVE){ + policyVertex.removeProperty(ATTR_POLICY_RESOURCES); policyEntity.setAttribute(ATTR_POLICY_RESOURCES, policyResources); context.addUpdated(policyEntity.getGuid(), policyEntity, entityType, policyVertex); } From cf7cb325a1fa5b84843a32ca8ae6dfe5e5396c38 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Thu, 25 Apr 2024 20:20:20 +0530 Subject: [PATCH 065/316] adding Persona to context --- .../datamesh/AbstractDomainPreProcessor.java | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index a6c4364708c..d6ad2581d87 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -27,13 +27,14 @@ import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.model.instance.AtlasObjectId; +import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; -import org.apache.atlas.repository.store.graph.v2.preprocessor.glossary.AbstractGlossaryPreProcessor; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; @@ -148,6 +149,11 @@ protected void updatePolicy(String currentQualifiedName, String updatedQualified AtlasEntity policyEntity = entityRetriever.toAtlasEntity(policyVertex); + if (policyEntity.hasRelationshipAttribute("accessControl")) { + AtlasVertex accessControl = entityRetriever.getEntityVertex(((AtlasObjectId) policyEntity.getRelationshipAttribute("accessControl")).getGuid()); + context.getDiscoveryContext().addResolvedGuid(GraphHelper.getGuid(accessControl), accessControl); + } + List policyResources = (List) policyEntity.getAttribute(ATTR_POLICY_RESOURCES); // Check if currentResource exists in the list before removing it if (policyResources.contains(currentResource)) { @@ -158,11 +164,9 @@ protected void updatePolicy(String currentQualifiedName, String updatedQualified LOG.info("CurrentResource {} not found in the policy resources. Skipping update.", currentResource); } - if(AtlasGraphUtilsV2.getState(policyVertex) == AtlasEntity.Status.ACTIVE){ - policyVertex.removeProperty(ATTR_POLICY_RESOURCES); - policyEntity.setAttribute(ATTR_POLICY_RESOURCES, policyResources); - context.addUpdated(policyEntity.getGuid(), policyEntity, entityType, policyVertex); - } + policyVertex.removeProperty(ATTR_POLICY_RESOURCES); + policyEntity.setAttribute(ATTR_POLICY_RESOURCES, policyResources); + context.addUpdated(policyEntity.getGuid(), policyEntity, entityType, policyVertex); } } From 91f661b48733bb9fc211c6523e2821bf4f96b53a Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Thu, 25 Apr 2024 21:12:26 +0530 Subject: [PATCH 066/316] logs to debug --- .../v2/preprocessor/datamesh/AbstractDomainPreProcessor.java | 1 + 1 file changed, 1 insertion(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index d6ad2581d87..8a30c8dd101 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -152,6 +152,7 @@ protected void updatePolicy(String currentQualifiedName, String updatedQualified if (policyEntity.hasRelationshipAttribute("accessControl")) { AtlasVertex accessControl = entityRetriever.getEntityVertex(((AtlasObjectId) policyEntity.getRelationshipAttribute("accessControl")).getGuid()); context.getDiscoveryContext().addResolvedGuid(GraphHelper.getGuid(accessControl), accessControl); + LOG.info("Resolved accessControl vertex: {}", accessControl); } List policyResources = (List) policyEntity.getAttribute(ATTR_POLICY_RESOURCES); From 39f37ed703860e9fdb82a338dd8a0b81d2b43629 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Fri, 26 Apr 2024 10:10:16 +0530 Subject: [PATCH 067/316] corrected variable names --- .../datamesh/AbstractDomainPreProcessor.java | 3 +-- .../preprocessor/datamesh/DataProductPreProcessor.java | 9 ++++----- .../v2/preprocessor/datamesh/DomainPreProcessor.java | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 8a30c8dd101..b0033307a65 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -31,7 +31,6 @@ import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; -import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; @@ -152,10 +151,10 @@ protected void updatePolicy(String currentQualifiedName, String updatedQualified if (policyEntity.hasRelationshipAttribute("accessControl")) { AtlasVertex accessControl = entityRetriever.getEntityVertex(((AtlasObjectId) policyEntity.getRelationshipAttribute("accessControl")).getGuid()); context.getDiscoveryContext().addResolvedGuid(GraphHelper.getGuid(accessControl), accessControl); - LOG.info("Resolved accessControl vertex: {}", accessControl); } List policyResources = (List) policyEntity.getAttribute(ATTR_POLICY_RESOURCES); + // Check if currentResource exists in the list before removing it if (policyResources.contains(currentResource)) { policyResources.remove(currentResource); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 6692845765a..fa857cd47d3 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -9,7 +9,6 @@ import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; -import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; @@ -131,11 +130,11 @@ private void processMoveDataProductToAnotherDomain(AtlasEntity product, AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("processMoveDataProductToAnotherDomain"); try { - String domainName = (String) product.getAttribute(NAME); + String productName = (String) product.getAttribute(NAME); - LOG.info("Moving dataProduct {} to Domain {}", domainName, targetDomainQualifiedName); + LOG.info("Moving dataProduct {} to Domain {}", productName, targetDomainQualifiedName); - productExists(domainName, targetDomainQualifiedName); + productExists(productName, targetDomainQualifiedName); String updatedQualifiedName = currentDataProductQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); @@ -146,7 +145,7 @@ private void processMoveDataProductToAnotherDomain(AtlasEntity product, //Update policy updatePolicy(currentDataProductQualifiedName, updatedQualifiedName, context); - LOG.info("Moved dataProduct {} to Domain {}", domainName, targetDomainQualifiedName); + LOG.info("Moved dataProduct {} to Domain {}", productName, targetDomainQualifiedName); } finally { RequestContext.get().endMetricRecord(recorder); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index 318e5251795..efadeaf23de 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -33,7 +33,6 @@ import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; -import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; @@ -176,6 +175,7 @@ private void processMoveSubDomainToAnotherDomain(AtlasEntity domain, // Move Sub-Domain as root Domain if(targetDomainQualifiedName.isEmpty()){ + LOG.info("Moving subDomain {} as root Domain", domainName); targetDomainQualifiedName = "default"; updatedQualifiedName = currentSubDomainQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); domain.setAttribute(QUALIFIED_NAME, updatedQualifiedName); From 4cd524d6db907106aaad0cee4c2f924a9b52cf54 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Fri, 26 Apr 2024 16:14:15 +0530 Subject: [PATCH 068/316] Change datasource to data_source in DataContract --- .../graph/v2/preprocessor/contract/ContractVersionUtils.java | 1 + .../store/graph/v2/preprocessor/contract/DataContract.java | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java index 149ec2b6ea7..3c25937aef0 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java @@ -54,6 +54,7 @@ private void extractAllVersions() throws AtlasBaseException { dsl.put("query", mapOf("bool", mapOf("must", mustClauseList))); Set attributes = new HashSet<>(); attributes.add(ATTR_CONTRACT); + attributes.add(ATTR_CERTIFICATE_STATUS); indexSearchParams.setDsl(dsl); indexSearchParams.setAttributes(attributes); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java index d96c885c4fc..ebb41445300 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java @@ -21,7 +21,7 @@ @JsonIgnoreProperties(ignoreUnknown = true) -@JsonPropertyOrder({"kind", "status", "template_version", "datasource", "dataset", "type", "description", "owners", +@JsonPropertyOrder({"kind", "status", "template_version", "data_source", "dataset", "type", "description", "owners", "tags", "certificate", "columns"}) public class DataContract { @Valid @NotNull @@ -32,7 +32,7 @@ public class DataContract { @JsonProperty(value = "template_version", defaultValue = "0.0.1") public String templateVersion; @Valid @NotNull - public String datasource; + public String data_source; @Valid @NotNull public String dataset; @Valid @NotNull From 8ed00b3f9b32f726056debd54d021a2070cfbe30 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Sat, 27 Apr 2024 15:16:25 +0530 Subject: [PATCH 069/316] Merge create and update AuthPolicy for DataContract --- .../policies/bootstrap_entity_policies.json | 40 ++----------------- 1 file changed, 4 insertions(+), 36 deletions(-) diff --git a/addons/policies/bootstrap_entity_policies.json b/addons/policies/bootstrap_entity_policies.json index ac8629faeb9..70d9b075654 100644 --- a/addons/policies/bootstrap_entity_policies.json +++ b/addons/policies/bootstrap_entity_policies.json @@ -3005,40 +3005,6 @@ ] } }, - { - "typeName": "AuthPolicy", - "attributes": - { - "name": "CREATE_DATA_CONTRACT", - "qualifiedName": "CREATE_DATA_CONTRACT", - "policyCategory": "bootstrap", - "policySubCategory": "default", - "policyServiceName": "atlas", - "policyType": "allow", - "policyPriority": 1, - "policyUsers": - [], - "policyGroups": - [], - "policyRoles": - [ - "$admin", - "$member", - "$api-token-default-access" - ], - "policyResourceCategory": "ENTITY", - "policyResources": - [ - "entity-type:DataContract", - "entity-classification:*", - "entity:*" - ], - "policyActions": - [ - "entity-create" - ] - } - }, { "typeName": "AuthPolicy", "attributes": @@ -3077,8 +3043,9 @@ "typeName": "AuthPolicy", "attributes": { - "name": "UPDATE_DATA_CONTRACT", - "qualifiedName": "UPDATE_DATA_CONTRACT", + "name": "CU_DATA_CONTRACT", + "qualifiedName": "CU_DATA_CONTRACT", + "description": "cu allow for data contract", "policyCategory": "bootstrap", "policySubCategory": "default", "policyServiceName": "atlas", @@ -3103,6 +3070,7 @@ ], "policyActions": [ + "entity-create", "entity-update" ] } From 5434342a4af7fc6a7ae283636d85adced53e0041 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Sat, 27 Apr 2024 17:40:47 +0530 Subject: [PATCH 070/316] Update DataContract attributes from public to private and create getter for needed attributes --- .../contract/ContractPreProcessor.java | 20 +++--- .../preprocessor/contract/DataContract.java | 62 ++++++++++--------- 2 files changed, 42 insertions(+), 40 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index 17ee1223018..b6d333536d5 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -77,12 +77,10 @@ private void processUpdateContract(AtlasEntity entity, EntityMutationContext con String contractString = (String) entity.getAttribute(ATTR_CONTRACT); AtlasVertex vertex = context.getVertex(entity.getGuid()); AtlasEntity existingContractEntity = entityRetriever.toAtlasEntity(vertex); - // TODO: Check for qualifiedName to understand if a particular version is getting updated or duplicate contract in payload if (!isEqualContract(contractString, (String) existingContractEntity.getAttribute(ATTR_CONTRACT))) { // Update the same asset(entity) throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Can't update a specific version of contract"); } - // Add cases for update in status field and certificateStatus } private void processCreateContract(AtlasEntity entity, EntityMutationContext context) throws AtlasBaseException { /* @@ -104,8 +102,8 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con String contractString = (String) entity.getAttribute(ATTR_CONTRACT); DataContract contract = DataContract.deserialize(contractString); String datasetQName = contractQName.substring(0, contractQName.lastIndexOf('/')); - contractQName = String.format("%s/%s/%s", datasetQName, contract.type.name(), CONTRACT_QUALIFIED_NAME_SUFFIX); - AtlasEntityWithExtInfo associatedAsset = getAssociatedAsset(datasetQName, contract.type.name()); + contractQName = String.format("%s/%s/%s", datasetQName, contract.getType().name(), CONTRACT_QUALIFIED_NAME_SUFFIX); + AtlasEntityWithExtInfo associatedAsset = getAssociatedAsset(datasetQName, contract.getType().name()); authorizeContractCreateOrUpdate(entity, associatedAsset); @@ -219,13 +217,13 @@ private void contractAttributeSync(AtlasEntity entity, DataContract contract) th VERIFIED - stat -> VERIFIED > */ - if (Objects.equals(entity.getAttribute(ATTR_CERTIFICATE_STATUS), DataContract.STATUS.VERIFIED.name())) { - contract.setStatus(String.valueOf(DataContract.STATUS.VERIFIED)); - } else if (Objects.equals(contract.getStatus(), DataContract.STATUS.VERIFIED)) { - entity.setAttribute(ATTR_CERTIFICATE_STATUS, DataContract.STATUS.VERIFIED.name()); + if (Objects.equals(entity.getAttribute(ATTR_CERTIFICATE_STATUS), DataContract.Status.VERIFIED.name())) { + contract.setStatus(String.valueOf(DataContract.Status.VERIFIED)); + } else if (Objects.equals(contract.getStatus(), DataContract.Status.VERIFIED)) { + entity.setAttribute(ATTR_CERTIFICATE_STATUS, DataContract.Status.VERIFIED.name()); } else { - entity.setAttribute(ATTR_CERTIFICATE_STATUS, DataContract.STATUS.DRAFT); - contract.setStatus(String.valueOf(DataContract.STATUS.DRAFT)); + entity.setAttribute(ATTR_CERTIFICATE_STATUS, DataContract.Status.DRAFT); + contract.setStatus(String.valueOf(DataContract.Status.DRAFT)); } } @@ -243,7 +241,7 @@ private void datasetAttributeSync(EntityMutationContext context, AtlasEntity ass // Update relationship with contract entity.setRelationshipAttribute(REL_ATTR_LATEST_CONTRACT, getAtlasObjectId(contractAsset)); - if (Objects.equals(contractAsset.getAttribute(ATTR_CERTIFICATE_STATUS), DataContract.STATUS.VERIFIED.name()) ) { + if (Objects.equals(contractAsset.getAttribute(ATTR_CERTIFICATE_STATUS), DataContract.Status.VERIFIED.name()) ) { entity.setRelationshipAttribute(REL_ATTR_GOVERNED_ASSET_CERTIFIED, getAtlasObjectId(contractAsset)); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java index ebb41445300..f8af5d4a61f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java @@ -24,42 +24,49 @@ @JsonPropertyOrder({"kind", "status", "template_version", "data_source", "dataset", "type", "description", "owners", "tags", "certificate", "columns"}) public class DataContract { - @Valid @NotNull - public String kind; + private static final String KIND_VALUE = "DataContract"; + private static final Pattern versionPattern = Pattern.compile("^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$"); + private static final ObjectMapper objectMapper = new ObjectMapper(); - public STATUS status; + @Valid @NotNull + private String kind; + private Status status; @JsonProperty(value = "template_version", defaultValue = "0.0.1") - public String templateVersion; + private String templateVersion; @Valid @NotNull - public String data_source; + private String data_source; @Valid @NotNull - public String dataset; + private String dataset; @Valid @NotNull - public DATASET_TYPE type; - public String description; - public List owners; - public List tags; - public String certificate; + private DatasetType type; + private String description; + private List owners; + private List tags; + private String certificate; @Valid - public List columns; + private List columns; private Map unknownFields = new HashMap<>(); @JsonSetter("type") public void setType(String type) throws AtlasBaseException { try { - this.type = DATASET_TYPE.from(type); + this.type = DatasetType.from(type); } catch (IllegalArgumentException | AtlasBaseException ex) { - throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "type " + type + " is inappropriate. Accepted values: " + Arrays.toString(DATASET_TYPE.values())); + throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "type " + type + " is inappropriate. Accepted values: " + Arrays.toString(DatasetType.values())); } } - public enum DATASET_TYPE { + public DatasetType getType() { + return type; + } + + public enum DatasetType { @JsonProperty("Table") Table, @JsonProperty("View") View, @JsonProperty("MaterialisedView") MaterialisedView; - public static DATASET_TYPE from(String s) throws AtlasBaseException { + public static DatasetType from(String s) throws AtlasBaseException { switch (s.toLowerCase()) { case "table": @@ -73,16 +80,16 @@ public static DATASET_TYPE from(String s) throws AtlasBaseException { } } } - public STATUS getStatus() { + public Status getStatus() { return status; } @JsonSetter("status") public void setStatus(String status) throws AtlasBaseException { try { - this.status = STATUS.from(status); + this.status = Status.from(status); } catch (IllegalArgumentException ex) { - throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "status " + status + " is inappropriate. Accepted values: " + Arrays.toString(STATUS.values())); + throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "status " + status + " is inappropriate. Accepted values: " + Arrays.toString(Status.values())); } } @@ -96,11 +103,11 @@ public Map getUnknownFields() { return unknownFields; } - public enum STATUS { + public enum Status { @JsonProperty("DRAFT") DRAFT, @JsonProperty("VERIFIED") VERIFIED; - public static STATUS from(String s) { + public static Status from(String s) { if(StringUtils.isEmpty(s)) { return DRAFT; } @@ -119,7 +126,7 @@ public static STATUS from(String s) { @JsonSetter("kind") public void setKind(String kind) throws AtlasBaseException { - if (!"DataContract".equals(kind)) { + if (!KIND_VALUE.equals(kind)) { throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "kind " + kind + " is inappropriate."); } this.kind = kind; @@ -133,7 +140,6 @@ public void setTemplateVersion(String templateVersion) throws AtlasBaseException } private boolean isSemVer(String version) { - Pattern versionPattern = Pattern.compile("^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$"); Matcher matcher = versionPattern.matcher(version); return matcher.matches(); } @@ -159,13 +165,13 @@ public Map getUnknownFields() { @JsonPropertyOrder({"name", "description", "data_type"}) public static final class Column { @NotNull - public String name; + private String name; - public String description; + private String description; - public boolean is_primary; + private boolean is_primary; - public String data_type; + private String data_type; private Map unknownFields = new HashMap<>(); @JsonAnySetter @@ -185,7 +191,6 @@ public static DataContract deserialize(String contractString) throws AtlasBaseEx throw new AtlasBaseException(BAD_REQUEST, "Missing attribute: contract."); } - ObjectMapper objectMapper = new ObjectMapper(); objectMapper.enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS); DataContract contract; try { @@ -215,7 +220,6 @@ public void validate() throws AtlasBaseException { public static String serialize(DataContract contract) throws AtlasBaseException { try { - ObjectMapper objectMapper = new ObjectMapper(); objectMapper.enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS); return objectMapper.writeValueAsString(contract); } catch (JsonProcessingException ex) { From 09c500a0c1b59e7cbbaacb26aea704e8f4d05e82 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Sat, 27 Apr 2024 20:13:59 +0530 Subject: [PATCH 071/316] Add setter for private attributes and formatted the file --- .../preprocessor/contract/DataContract.java | 153 +++++++++++------- 1 file changed, 98 insertions(+), 55 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java index f8af5d4a61f..62887c3b238 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java @@ -31,36 +31,42 @@ public class DataContract { @Valid @NotNull private String kind; private Status status; - @JsonProperty(value = "template_version", defaultValue = "0.0.1") private String templateVersion; @Valid @NotNull - private String data_source; + private String data_source; @Valid @NotNull - private String dataset; + private String dataset; @Valid @NotNull - private DatasetType type; - private String description; - private List owners; - private List tags; - private String certificate; + private DatasetType type; + private String description; + private List owners; + private List tags; + private String certificate; @Valid - private List columns; - private Map unknownFields = new HashMap<>(); + private List columns; + private final Map unknownFields = new HashMap<>(); - @JsonSetter("type") - public void setType(String type) throws AtlasBaseException { - try { - this.type = DatasetType.from(type); - } catch (IllegalArgumentException | AtlasBaseException ex) { - throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "type " + type + " is inappropriate. Accepted values: " + Arrays.toString(DatasetType.values())); - } - } + public enum Status { + @JsonProperty("DRAFT") DRAFT, + @JsonProperty("VERIFIED") VERIFIED; - public DatasetType getType() { - return type; - } + public static Status from(String s) { + if(StringUtils.isEmpty(s)) { + return DRAFT; + } + switch (s.toLowerCase()) { + case "draft": + return DRAFT; + case "verified": + return VERIFIED; + + default: + return DRAFT; + } + } + } public enum DatasetType { @JsonProperty("Table") Table, @JsonProperty("View") View, @@ -80,22 +86,13 @@ public static DatasetType from(String s) throws AtlasBaseException { } } } + public Status getStatus() { return status; } - @JsonSetter("status") - public void setStatus(String status) throws AtlasBaseException { - try { - this.status = Status.from(status); - } catch (IllegalArgumentException ex) { - throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "status " + status + " is inappropriate. Accepted values: " + Arrays.toString(Status.values())); - } - } - - @JsonAnySetter - public void setUnknownFields(String key, Object value) { - unknownFields.put(key, value); + public DatasetType getType() { + return type; } @JsonAnyGetter @@ -103,27 +100,6 @@ public Map getUnknownFields() { return unknownFields; } - public enum Status { - @JsonProperty("DRAFT") DRAFT, - @JsonProperty("VERIFIED") VERIFIED; - - public static Status from(String s) { - if(StringUtils.isEmpty(s)) { - return DRAFT; - } - switch (s.toLowerCase()) { - case "draft": - return DRAFT; - - case "verified": - return VERIFIED; - - default: - return DRAFT; - } - } - } - @JsonSetter("kind") public void setKind(String kind) throws AtlasBaseException { if (!KIND_VALUE.equals(kind)) { @@ -132,6 +108,15 @@ public void setKind(String kind) throws AtlasBaseException { this.kind = kind; } + @JsonSetter("status") + public void setStatus(String status) throws AtlasBaseException { + try { + this.status = Status.from(status); + } catch (IllegalArgumentException ex) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "status " + status + " is inappropriate. Accepted values: " + Arrays.toString(Status.values())); + } + } + public void setTemplateVersion(String templateVersion) throws AtlasBaseException { if (!isSemVer(templateVersion)) { throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "Invalid template_version syntax"); @@ -139,6 +124,49 @@ public void setTemplateVersion(String templateVersion) throws AtlasBaseException this.templateVersion = templateVersion; } + public void setDataSource(String data_source) { + this.data_source = data_source; + } + + public void setDataset(String dataset) { + this.dataset = dataset; + } + + @JsonSetter("type") + public void setType(String type) throws AtlasBaseException { + try { + this.type = DatasetType.from(type); + } catch (IllegalArgumentException | AtlasBaseException ex) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "type " + type + " is inappropriate. Accepted values: " + Arrays.toString(DatasetType.values())); + } + } + + public void setDescription(String description) { + this.description = description; + } + + public void setOwners(List owners) { + this.owners = owners; + } + + public void setTags(List tags) { + this.tags = tags; + } + + public void setCertificate(String certificate) { + this.certificate = certificate; + } + + public void setColumns(List columns) { + this.columns = columns; + } + + @JsonAnySetter + public void setUnknownFields(String key, Object value) { + unknownFields.put(key, value); + } + + private boolean isSemVer(String version) { Matcher matcher = versionPattern.matcher(version); return matcher.matches(); @@ -161,6 +189,7 @@ public Map getUnknownFields() { } } + @JsonIgnoreProperties(ignoreUnknown = true) @JsonPropertyOrder({"name", "description", "data_type"}) public static final class Column { @@ -183,6 +212,21 @@ public Map getUnknownFields() { return unknownFields; } + public void setName(String name) { + this.name = name; + } + + public void setDescription(String description) { + this.description = description; + } + + public void setIs_primary(boolean is_primary) { + this.is_primary = is_primary; + } + + public void setData_type(String data_type) { + this.data_type = data_type; + } } public static DataContract deserialize(String contractString) throws AtlasBaseException { @@ -227,6 +271,5 @@ public static String serialize(DataContract contract) throws AtlasBaseException } } - } From d78cdc14ad043336599c636b41c5454cd8c966dd Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Sun, 28 Apr 2024 09:45:57 +0530 Subject: [PATCH 072/316] Address PR review changes * Remove version util and moved getCurrentVersion method to preprocessor * Remove unused varibles * Optimize fetching latest version of contract * --- .../apache/atlas/repository/Constants.java | 2 + .../AbstractContractPreProcessor.java | 2 - .../contract/ContractPreProcessor.java | 55 +++++++++--- .../contract/ContractVersionUtils.java | 85 ------------------- 4 files changed, 46 insertions(+), 98 deletions(-) delete mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 2152921981c..8df6e0bbaa4 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -167,6 +167,8 @@ public final class Constants { * Contract */ public static final String CONTRACT_ENTITY_TYPE = "DataContract"; + public static final String ATTR_CONTRACT_VERSION = "dataContractVersion"; + /** * Lineage relations. diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java index 9b063d38f5a..e167a00b9a1 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java @@ -79,8 +79,6 @@ AtlasEntity.AtlasEntityWithExtInfo getAssociatedAsset(String datasetQName, Strin AtlasVertex entityVertex = AtlasGraphUtilsV2.getVertexByUniqueAttributes(graph, entityType, uniqAttributes); - EntityGraphRetriever entityRetriever = new EntityGraphRetriever(graph, typeRegistry, true); - AtlasEntity.AtlasEntityWithExtInfo ret = entityRetriever.toAtlasEntityWithExtInfo(entityVertex); if (ret == null) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index b6d333536d5..a60ed4ef09c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -7,6 +7,8 @@ import org.apache.atlas.RequestContext; import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.discovery.AtlasSearchResult; +import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; import org.apache.atlas.model.instance.AtlasStruct; @@ -25,14 +27,12 @@ import java.util.*; import static org.apache.atlas.AtlasErrorCode.*; -import static org.apache.atlas.repository.Constants.ATTR_CERTIFICATE_STATUS; -import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; -import static org.apache.atlas.repository.Constants.ATTR_CONTRACT; +import static org.apache.atlas.repository.Constants.*; +import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; import static org.apache.atlas.type.AtlasTypeUtil.getAtlasObjectId; public class ContractPreProcessor extends AbstractContractPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(ContractPreProcessor.class); - public static final String ATTR_VERSION = "dataContractVersion"; public static final String ATTR_ASSET_GUID = "dataContractAssetGuid"; public static final String REL_ATTR_LATEST_CONTRACT = "dataContractLatest"; public static final String REL_ATTR_GOVERNED_ASSET_CERTIFIED = "dataContractLatestCertified"; @@ -45,7 +45,7 @@ public class ContractPreProcessor extends AbstractContractPreProcessor { public static final String CONTRACT_ATTR_STATUS = "status"; private final AtlasEntityStore entityStore; private final boolean storeDifferentialAudits; - private EntityDiscoveryService discovery; + private final EntityDiscoveryService discovery; @@ -107,12 +107,13 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con authorizeContractCreateOrUpdate(entity, associatedAsset); - contractAttributeSync(entity, contract); - contractString = DataContract.serialize(contract); + boolean contractSync = syncContractCertificateStatus(entity, contract); + if (contractSync) { + contractString = DataContract.serialize(contract); + } entity.setAttribute(ATTR_CONTRACT, contractString); - ContractVersionUtils versionUtil = new ContractVersionUtils(contractQName, context, entityRetriever, typeRegistry, entityStore, graph, discovery); - AtlasEntity currentVersionEntity = versionUtil.getCurrentVersion(); + AtlasEntity currentVersionEntity = getCurrentVersion(contractQName); int newVersionNumber = 1; if (currentVersionEntity != null) { // Contract already exist @@ -140,7 +141,7 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con } } entity.setAttribute(QUALIFIED_NAME, String.format("%s/V%s", contractQName, newVersionNumber)); - entity.setAttribute(ATTR_VERSION, newVersionNumber); + entity.setAttribute(ATTR_CONTRACT_VERSION, newVersionNumber); entity.setAttribute(ATTR_ASSET_GUID, associatedAsset.getEntity().getGuid()); datasetAttributeSync(context, associatedAsset.getEntity(), contract, entity); @@ -191,6 +192,34 @@ private void updateExistingVersion(EntityMutationContext context, AtlasEntity en } + public AtlasEntity getCurrentVersion(String contractQName) throws AtlasBaseException { + IndexSearchParams indexSearchParams = new IndexSearchParams(); + Map dsl = new HashMap<>(); + int size = 1; + + List> mustClauseList = new ArrayList<>(); + mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", CONTRACT_ENTITY_TYPE))); + mustClauseList.add(mapOf("wildcard", mapOf(QUALIFIED_NAME, String.format("%s/*", contractQName)))); + + dsl.put("query", mapOf("bool", mapOf("must", mustClauseList))); + dsl.put("sort", Collections.singletonList(mapOf(ATTR_CONTRACT_VERSION, mapOf("order", "desc")))); + dsl.put("size", size); + + final Set attributes = new HashSet<>(); + attributes.add(ATTR_CONTRACT); + attributes.add(ATTR_CERTIFICATE_STATUS); + + indexSearchParams.setDsl(dsl); + indexSearchParams.setAttributes(attributes); + indexSearchParams.setSuppressLogs(true); + + AtlasSearchResult result = discovery.directIndexSearch(indexSearchParams); + if (result == null) { + return null; + } + return new AtlasEntity(result.getEntities().get(0)); + } + private void removeCreatingVertex(EntityMutationContext context, AtlasEntity entity) throws AtlasBaseException { context.getCreatedEntities().remove(entity); try { @@ -204,7 +233,8 @@ private void removeCreatingVertex(EntityMutationContext context, AtlasEntity ent } - private void contractAttributeSync(AtlasEntity entity, DataContract contract) throws AtlasBaseException { + private boolean syncContractCertificateStatus(AtlasEntity entity, DataContract contract) throws AtlasBaseException { + boolean contractSync = false; // Sync certificateStatus if (!Objects.equals(entity.getAttribute(ATTR_CERTIFICATE_STATUS), contract.getStatus().name())) { /* @@ -219,14 +249,17 @@ private void contractAttributeSync(AtlasEntity entity, DataContract contract) th */ if (Objects.equals(entity.getAttribute(ATTR_CERTIFICATE_STATUS), DataContract.Status.VERIFIED.name())) { contract.setStatus(String.valueOf(DataContract.Status.VERIFIED)); + contractSync = true; } else if (Objects.equals(contract.getStatus(), DataContract.Status.VERIFIED)) { entity.setAttribute(ATTR_CERTIFICATE_STATUS, DataContract.Status.VERIFIED.name()); } else { entity.setAttribute(ATTR_CERTIFICATE_STATUS, DataContract.Status.DRAFT); contract.setStatus(String.valueOf(DataContract.Status.DRAFT)); + contractSync = true; } } + return contractSync; } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java deleted file mode 100644 index 3c25937aef0..00000000000 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java +++ /dev/null @@ -1,85 +0,0 @@ -package org.apache.atlas.repository.store.graph.v2.preprocessor.contract; - -import org.apache.atlas.discovery.EntityDiscoveryService; -import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.discovery.AtlasSearchResult; -import org.apache.atlas.model.discovery.IndexSearchParams; -import org.apache.atlas.model.instance.AtlasEntity; -import org.apache.atlas.model.instance.AtlasEntityHeader; -import org.apache.atlas.repository.graphdb.AtlasGraph; -import org.apache.atlas.repository.store.graph.AtlasEntityStore; -import org.apache.atlas.repository.store.graph.v2.*; -import org.apache.atlas.type.AtlasTypeRegistry; - -import java.util.*; - -import static org.apache.atlas.repository.Constants.*; -import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; - - -public class ContractVersionUtils { - - private final EntityMutationContext context; - public final EntityGraphRetriever entityRetriever; - private final AtlasTypeRegistry atlasTypeRegistry; - private AtlasEntityStore entityStore; - private String contractQName; - - public final AtlasGraph graph; - private List versionList; - private EntityDiscoveryService discovery; - - public ContractVersionUtils(String contractQName, EntityMutationContext context, EntityGraphRetriever entityRetriever, - AtlasTypeRegistry atlasTypeRegistry, AtlasEntityStore entityStore, AtlasGraph graph, - EntityDiscoveryService discovery) { - this.context = context; - this.entityRetriever = entityRetriever; - this.atlasTypeRegistry = atlasTypeRegistry; - this.graph = graph; - this.entityStore = entityStore; - this.contractQName = contractQName; - this.discovery = discovery; - } - - private void extractAllVersions() throws AtlasBaseException { - List ret = new ArrayList<>(); - - IndexSearchParams indexSearchParams = new IndexSearchParams(); - Map dsl = new HashMap<>(); - - List mustClauseList = new ArrayList(); - mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", CONTRACT_ENTITY_TYPE))); - mustClauseList.add(mapOf("wildcard", mapOf(QUALIFIED_NAME, String.format("%s/*", contractQName)))); - - dsl.put("query", mapOf("bool", mapOf("must", mustClauseList))); - Set attributes = new HashSet<>(); - attributes.add(ATTR_CONTRACT); - attributes.add(ATTR_CERTIFICATE_STATUS); - - indexSearchParams.setDsl(dsl); - indexSearchParams.setAttributes(attributes); - indexSearchParams.setSuppressLogs(true); - - AtlasSearchResult result = discovery.directIndexSearch(indexSearchParams); - if (result != null) { - ret = result.getEntities(); - } - this.versionList = ret; - } - - public AtlasEntity getCurrentVersion() throws AtlasBaseException { - if (this.versionList == null) { - extractAllVersions(); - } - if (this.versionList == null) { - return null; - } - Collections.sort(this.versionList, (e1, e2) -> { - String e1QName = (String) e1.getAttribute(QUALIFIED_NAME); - String e2QName = (String) e2.getAttribute(QUALIFIED_NAME); - - return e2QName.compareTo(e1QName); - }); - return new AtlasEntity(this.versionList.get(0)); - } -} From 67855310f2f3b79b64c540982c01728003a05a9e Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Mon, 29 Apr 2024 13:13:32 +0530 Subject: [PATCH 073/316] added custom attributes --- .../v2/preprocessor/datamesh/DataProductPreProcessor.java | 3 +++ .../graph/v2/preprocessor/datamesh/DomainPreProcessor.java | 3 +++ 2 files changed, 6 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index fa857cd47d3..5da61debd79 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -61,11 +61,14 @@ private void processCreateProduct(AtlasEntity entity, AtlasVertex vertex) throws AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreateProduct"); String productName = (String) entity.getAttribute(NAME); String parentDomainQualifiedName = (String) entity.getAttribute(PARENT_DOMAIN_QN); + Map customAttributes = new HashMap<>(); + customAttributes.put("isQualifiedNameMigrated", "true"); productExists(productName, parentDomainQualifiedName); String newQualifiedName = createQualifiedName(parentDomainQualifiedName); entity.setAttribute(QUALIFIED_NAME, newQualifiedName); + entity.setCustomAttributes(customAttributes); RequestContext.get().endMetricRecord(metricRecorder); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index efadeaf23de..6abf2df32d8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -90,9 +90,12 @@ private void processCreateDomain(AtlasEntity entity, AtlasVertex vertex) throws AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreateDomain"); String domainName = (String) entity.getAttribute(NAME); String parentDomainQualifiedName = (String) entity.getAttribute(PARENT_DOMAIN_QN); + Map customAttributes = new HashMap<>(); + customAttributes.put("isQualifiedNameMigrated", "true"); domainExists(domainName, parentDomainQualifiedName); entity.setAttribute(QUALIFIED_NAME, createQualifiedName(parentDomainQualifiedName)); + entity.setCustomAttributes(customAttributes); RequestContext.get().endMetricRecord(metricRecorder); } From 9779ffeb081bbbb4ae6f0bbea7b443fcc633b906 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Mon, 29 Apr 2024 22:03:43 +0530 Subject: [PATCH 074/316] Address code review comments * Remove purgebyId call and remove skip auth in the purge method * Make DataContract attributes public for serialisation and deserialisation of JSON * Make minor changes for optimisation --- .../store/graph/v2/AtlasEntityStoreV2.java | 5 +- .../AbstractContractPreProcessor.java | 8 +-- .../contract/ContractPreProcessor.java | 33 +++++------ .../preprocessor/contract/DataContract.java | 57 +++++++------------ 4 files changed, 38 insertions(+), 65 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 8e11aaeff1b..519068a9da5 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -695,9 +695,8 @@ public EntityMutationResponse purgeByIds(Set guids) throws AtlasBaseExce if (CollectionUtils.isEmpty(guids)) { throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "Guid(s) not specified"); } - if (!RequestContext.get().isSkipAuthorizationCheck()) { - AtlasAuthorizationUtils.verifyAccess(new AtlasAdminAccessRequest(AtlasPrivilege.ADMIN_PURGE), "purge entity: guids=", guids); - } + + AtlasAuthorizationUtils.verifyAccess(new AtlasAdminAccessRequest(AtlasPrivilege.ADMIN_PURGE), "purge entity: guids=", guids); Collection purgeCandidates = new ArrayList<>(); for (String guid : guids) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java index e167a00b9a1..0a4521e34b6 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java @@ -79,13 +79,7 @@ AtlasEntity.AtlasEntityWithExtInfo getAssociatedAsset(String datasetQName, Strin AtlasVertex entityVertex = AtlasGraphUtilsV2.getVertexByUniqueAttributes(graph, entityType, uniqAttributes); - AtlasEntity.AtlasEntityWithExtInfo ret = entityRetriever.toAtlasEntityWithExtInfo(entityVertex); - - if (ret == null) { - throw new AtlasBaseException(INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND, entityType.getTypeName(), - uniqAttributes.toString()); - } - return ret; + return entityRetriever.toAtlasEntityWithExtInfo(entityVertex); } AtlasEntityType ensureEntityType(String typeName) throws AtlasBaseException { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index a60ed4ef09c..f45e6ecf574 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -19,6 +19,7 @@ import org.apache.atlas.repository.store.graph.v2.*; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.commons.collections.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,6 +48,7 @@ public class ContractPreProcessor extends AbstractContractPreProcessor { private final boolean storeDifferentialAudits; private final EntityDiscoveryService discovery; + private final AtlasEntityComparator entityComparator; public ContractPreProcessor(AtlasGraph graph, AtlasTypeRegistry typeRegistry, @@ -57,6 +59,8 @@ public ContractPreProcessor(AtlasGraph graph, AtlasTypeRegistry typeRegistry, this.storeDifferentialAudits = storeDifferentialAudits; this.entityStore = entityStore; this.discovery = discovery; + this.entityComparator = new AtlasEntityComparator(typeRegistry, entityRetriever, null, true, true); + } @Override @@ -110,15 +114,15 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con boolean contractSync = syncContractCertificateStatus(entity, contract); if (contractSync) { contractString = DataContract.serialize(contract); + entity.setAttribute(ATTR_CONTRACT, contractString); } - entity.setAttribute(ATTR_CONTRACT, contractString); - AtlasEntity currentVersionEntity = getCurrentVersion(contractQName); - int newVersionNumber = 1; + AtlasEntity currentVersionEntity = getCurrentVersion(associatedAsset.getEntity().getGuid()); + Long newVersionNumber = 1L; if (currentVersionEntity != null) { // Contract already exist String qName = (String) currentVersionEntity.getAttribute(QUALIFIED_NAME); - int currentVersionNumber = Integer.parseInt(qName.substring(qName.lastIndexOf("/V") + 2)); + Long currentVersionNumber = (Long) currentVersionEntity.getAttribute(ATTR_CONTRACT_VERSION); List attributes = getDiffAttributes(context, entity, currentVersionEntity); if (attributes.isEmpty()) { // No changes in the contract, Not creating new version @@ -149,7 +153,6 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con } private List getDiffAttributes(EntityMutationContext context, AtlasEntity entity, AtlasEntity latestExistingVersion) throws AtlasBaseException { - AtlasEntityComparator entityComparator = new AtlasEntityComparator(typeRegistry, entityRetriever, context.getGuidAssignments(), true, true); AtlasEntityComparator.AtlasEntityDiffResult diffResult = entityComparator.getDiffResult(entity, latestExistingVersion, false); List attributesSet = new ArrayList<>(); @@ -183,23 +186,21 @@ private void updateExistingVersion(EntityMutationContext context, AtlasEntity en removeCreatingVertex(context, entity); entity.setAttribute(QUALIFIED_NAME, currentVersionEntity.getAttribute(QUALIFIED_NAME)); entity.setGuid(currentVersionEntity.getGuid()); - - AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(entity.getGuid()); - + AtlasVertex vertex = context.getVertex(entity.getGuid()); AtlasEntityType entityType = ensureEntityType(entity.getTypeName()); context.addUpdated(entity.getGuid(), entity, entityType, vertex); } - public AtlasEntity getCurrentVersion(String contractQName) throws AtlasBaseException { + public AtlasEntity getCurrentVersion(String datasetGuid) throws AtlasBaseException { IndexSearchParams indexSearchParams = new IndexSearchParams(); Map dsl = new HashMap<>(); int size = 1; List> mustClauseList = new ArrayList<>(); mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", CONTRACT_ENTITY_TYPE))); - mustClauseList.add(mapOf("wildcard", mapOf(QUALIFIED_NAME, String.format("%s/*", contractQName)))); + mustClauseList.add(mapOf("term", mapOf(ATTR_ASSET_GUID, datasetGuid))); dsl.put("query", mapOf("bool", mapOf("must", mustClauseList))); dsl.put("sort", Collections.singletonList(mapOf(ATTR_CONTRACT_VERSION, mapOf("order", "desc")))); @@ -208,13 +209,14 @@ public AtlasEntity getCurrentVersion(String contractQName) throws AtlasBaseExcep final Set attributes = new HashSet<>(); attributes.add(ATTR_CONTRACT); attributes.add(ATTR_CERTIFICATE_STATUS); + attributes.add(ATTR_CONTRACT_VERSION); indexSearchParams.setDsl(dsl); indexSearchParams.setAttributes(attributes); indexSearchParams.setSuppressLogs(true); AtlasSearchResult result = discovery.directIndexSearch(indexSearchParams); - if (result == null) { + if (result == null || CollectionUtils.isEmpty(result.getEntities())) { return null; } return new AtlasEntity(result.getEntities().get(0)); @@ -222,14 +224,7 @@ public AtlasEntity getCurrentVersion(String contractQName) throws AtlasBaseExcep private void removeCreatingVertex(EntityMutationContext context, AtlasEntity entity) throws AtlasBaseException { context.getCreatedEntities().remove(entity); - try { - RequestContext.get().setSkipAuthorizationCheck(true); - Set guids = new HashSet<>(); - guids.add(entity.getGuid()); - entityStore.purgeByIds(guids); - } finally { - RequestContext.get().setSkipAuthorizationCheck(false); - } + graph.removeVertex(context.getVertex(entity.getGuid())); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java index 62887c3b238..ae45bf2353e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java @@ -27,24 +27,27 @@ public class DataContract { private static final String KIND_VALUE = "DataContract"; private static final Pattern versionPattern = Pattern.compile("^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$"); private static final ObjectMapper objectMapper = new ObjectMapper(); + static { + objectMapper.enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS); + } @Valid @NotNull - private String kind; - private Status status; + public String kind; + public Status status; @JsonProperty(value = "template_version", defaultValue = "0.0.1") - private String templateVersion; + public String templateVersion; @Valid @NotNull - private String data_source; + public String data_source; @Valid @NotNull - private String dataset; + public String dataset; @Valid @NotNull - private DatasetType type; - private String description; - private List owners; - private List tags; - private String certificate; + public DatasetType type; + public String description; + public List owners; + public List tags; + public String certificate; @Valid - private List columns; + public List columns; private final Map unknownFields = new HashMap<>(); public enum Status { @@ -124,6 +127,7 @@ public void setTemplateVersion(String templateVersion) throws AtlasBaseException this.templateVersion = templateVersion; } + @JsonSetter("data_source") public void setDataSource(String data_source) { this.data_source = data_source; } @@ -132,7 +136,6 @@ public void setDataset(String dataset) { this.dataset = dataset; } - @JsonSetter("type") public void setType(String type) throws AtlasBaseException { try { this.type = DatasetType.from(type); @@ -177,7 +180,7 @@ private boolean isSemVer(String version) { public static final class BusinessTag { @NotNull public String name; - private Map unknownFields = new HashMap<>(); + private final Map unknownFields = new HashMap<>(); @JsonAnySetter public void setUnknownFields(String key, Object value) { @@ -194,14 +197,14 @@ public Map getUnknownFields() { @JsonPropertyOrder({"name", "description", "data_type"}) public static final class Column { @NotNull - private String name; + public String name; - private String description; + public String description; - private boolean is_primary; + public boolean is_primary; - private String data_type; - private Map unknownFields = new HashMap<>(); + public String data_type; + private final Map unknownFields = new HashMap<>(); @JsonAnySetter public void setUnknownFields(String key, Object value) { @@ -211,22 +214,6 @@ public void setUnknownFields(String key, Object value) { public Map getUnknownFields() { return unknownFields; } - - public void setName(String name) { - this.name = name; - } - - public void setDescription(String description) { - this.description = description; - } - - public void setIs_primary(boolean is_primary) { - this.is_primary = is_primary; - } - - public void setData_type(String data_type) { - this.data_type = data_type; - } } public static DataContract deserialize(String contractString) throws AtlasBaseException { @@ -235,7 +222,6 @@ public static DataContract deserialize(String contractString) throws AtlasBaseEx throw new AtlasBaseException(BAD_REQUEST, "Missing attribute: contract."); } - objectMapper.enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS); DataContract contract; try { contract = objectMapper.readValue(contractString, DataContract.class); @@ -264,7 +250,6 @@ public void validate() throws AtlasBaseException { public static String serialize(DataContract contract) throws AtlasBaseException { try { - objectMapper.enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS); return objectMapper.writeValueAsString(contract); } catch (JsonProcessingException ex) { throw new AtlasBaseException(JSON_ERROR, ex.getMessage()); From 191b447418c189f5bfc75e4820a2a98cc787889f Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Tue, 30 Apr 2024 08:22:51 +0530 Subject: [PATCH 075/316] Remove redundant code/arguments in contract preprocessor --- .../store/graph/v2/AtlasEntityStoreV2.java | 2 +- .../contract/ContractPreProcessor.java | 33 ++++++++----------- 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 519068a9da5..3436ec3e107 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -1838,7 +1838,7 @@ public PreProcessor getPreProcessor(String typeName) { break; case CONTRACT_ENTITY_TYPE: - preProcessor = new ContractPreProcessor(graph, typeRegistry, entityRetriever, this, storeDifferentialAudits, discovery); + preProcessor = new ContractPreProcessor(graph, typeRegistry, entityRetriever, storeDifferentialAudits, discovery); break; } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index f45e6ecf574..47fc097042c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -15,7 +15,6 @@ import org.apache.atlas.model.instance.EntityMutations; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; -import org.apache.atlas.repository.store.graph.AtlasEntityStore; import org.apache.atlas.repository.store.graph.v2.*; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; @@ -39,12 +38,14 @@ public class ContractPreProcessor extends AbstractContractPreProcessor { public static final String REL_ATTR_GOVERNED_ASSET_CERTIFIED = "dataContractLatestCertified"; public static final String REL_ATTR_PREVIOUS_VERSION = "dataContractPreviousVersion"; public static final String ASSET_ATTR_HAS_CONTRACT = "hasContract"; - public static final String ASSET_ATTR_DESCRIPTION = "description"; - public static final String CONTRACT_QUALIFIED_NAME_SUFFIX = "contract"; - public static final String VERSION_PREFIX = "version"; public static final String CONTRACT_ATTR_STATUS = "status"; - private final AtlasEntityStore entityStore; + private static final Set contractAttributes = new HashSet<>(); + static { + contractAttributes.add(ATTR_CONTRACT); + contractAttributes.add(ATTR_CERTIFICATE_STATUS); + contractAttributes.add(ATTR_CONTRACT_VERSION); + } private final boolean storeDifferentialAudits; private final EntityDiscoveryService discovery; @@ -52,12 +53,11 @@ public class ContractPreProcessor extends AbstractContractPreProcessor { public ContractPreProcessor(AtlasGraph graph, AtlasTypeRegistry typeRegistry, - EntityGraphRetriever entityRetriever, AtlasEntityStore entityStore, + EntityGraphRetriever entityRetriever, boolean storeDifferentialAudits, EntityDiscoveryService discovery) { super(graph, typeRegistry, entityRetriever); this.storeDifferentialAudits = storeDifferentialAudits; - this.entityStore = entityStore; this.discovery = discovery; this.entityComparator = new AtlasEntityComparator(typeRegistry, entityRetriever, null, true, true); @@ -121,9 +121,8 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con Long newVersionNumber = 1L; if (currentVersionEntity != null) { // Contract already exist - String qName = (String) currentVersionEntity.getAttribute(QUALIFIED_NAME); Long currentVersionNumber = (Long) currentVersionEntity.getAttribute(ATTR_CONTRACT_VERSION); - List attributes = getDiffAttributes(context, entity, currentVersionEntity); + List attributes = getDiffAttributes(entity, currentVersionEntity); if (attributes.isEmpty()) { // No changes in the contract, Not creating new version removeCreatingVertex(context, entity); @@ -148,11 +147,11 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con entity.setAttribute(ATTR_CONTRACT_VERSION, newVersionNumber); entity.setAttribute(ATTR_ASSET_GUID, associatedAsset.getEntity().getGuid()); - datasetAttributeSync(context, associatedAsset.getEntity(), contract, entity); + datasetAttributeSync(context, associatedAsset.getEntity(), entity); } - private List getDiffAttributes(EntityMutationContext context, AtlasEntity entity, AtlasEntity latestExistingVersion) throws AtlasBaseException { + private List getDiffAttributes(AtlasEntity entity, AtlasEntity latestExistingVersion) throws AtlasBaseException { AtlasEntityComparator.AtlasEntityDiffResult diffResult = entityComparator.getDiffResult(entity, latestExistingVersion, false); List attributesSet = new ArrayList<>(); @@ -206,13 +205,8 @@ public AtlasEntity getCurrentVersion(String datasetGuid) throws AtlasBaseExcepti dsl.put("sort", Collections.singletonList(mapOf(ATTR_CONTRACT_VERSION, mapOf("order", "desc")))); dsl.put("size", size); - final Set attributes = new HashSet<>(); - attributes.add(ATTR_CONTRACT); - attributes.add(ATTR_CERTIFICATE_STATUS); - attributes.add(ATTR_CONTRACT_VERSION); - indexSearchParams.setDsl(dsl); - indexSearchParams.setAttributes(attributes); + indexSearchParams.setAttributes(contractAttributes); indexSearchParams.setSuppressLogs(true); AtlasSearchResult result = discovery.directIndexSearch(indexSearchParams); @@ -222,10 +216,9 @@ public AtlasEntity getCurrentVersion(String datasetGuid) throws AtlasBaseExcepti return new AtlasEntity(result.getEntities().get(0)); } - private void removeCreatingVertex(EntityMutationContext context, AtlasEntity entity) throws AtlasBaseException { + private void removeCreatingVertex(EntityMutationContext context, AtlasEntity entity) { context.getCreatedEntities().remove(entity); graph.removeVertex(context.getVertex(entity.getGuid())); - } private boolean syncContractCertificateStatus(AtlasEntity entity, DataContract contract) throws AtlasBaseException { @@ -258,7 +251,7 @@ private boolean syncContractCertificateStatus(AtlasEntity entity, DataContract c } - private void datasetAttributeSync(EntityMutationContext context, AtlasEntity associatedAsset, DataContract contract, AtlasEntity contractAsset) throws AtlasBaseException { + private void datasetAttributeSync(EntityMutationContext context, AtlasEntity associatedAsset, AtlasEntity contractAsset) throws AtlasBaseException { // Creating new empty AtlasEntity to update with selective attributes only AtlasEntity entity = new AtlasEntity(associatedAsset.getTypeName()); entity.setGuid(associatedAsset.getGuid()); From e4e39a3b1c6671c4c25cfe7ac87a61175bda1935 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Tue, 30 Apr 2024 11:14:29 +0530 Subject: [PATCH 076/316] created static variable for custom attribute --- .../store/graph/v2/preprocessor/PreProcessorUtils.java | 1 + .../graph/v2/preprocessor/datamesh/DataProductPreProcessor.java | 2 +- .../graph/v2/preprocessor/datamesh/DomainPreProcessor.java | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index 57fd775d4af..c358e40027c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -38,6 +38,7 @@ public class PreProcessorUtils { public static final String PARENT_DOMAIN_QN = "parentDomainQualifiedName"; public static final String SUPER_DOMAIN_QN = "superDomainQualifiedName"; public static final String DATA_DOMAIN = "dataDomain"; + public static final String MIGRATION_CUSTOM_ATTRIBUTE = "isQualifiedNameMigrated"; //Query models constants public static final String PREFIX_QUERY_QN = "default/collection/"; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 5da61debd79..4b84a1b450b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -62,7 +62,7 @@ private void processCreateProduct(AtlasEntity entity, AtlasVertex vertex) throws String productName = (String) entity.getAttribute(NAME); String parentDomainQualifiedName = (String) entity.getAttribute(PARENT_DOMAIN_QN); Map customAttributes = new HashMap<>(); - customAttributes.put("isQualifiedNameMigrated", "true"); + customAttributes.put(MIGRATION_CUSTOM_ATTRIBUTE, "true"); productExists(productName, parentDomainQualifiedName); String newQualifiedName = createQualifiedName(parentDomainQualifiedName); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index 6abf2df32d8..d45dab7fdc7 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -91,7 +91,7 @@ private void processCreateDomain(AtlasEntity entity, AtlasVertex vertex) throws String domainName = (String) entity.getAttribute(NAME); String parentDomainQualifiedName = (String) entity.getAttribute(PARENT_DOMAIN_QN); Map customAttributes = new HashMap<>(); - customAttributes.put("isQualifiedNameMigrated", "true"); + customAttributes.put(MIGRATION_CUSTOM_ATTRIBUTE, "true"); domainExists(domainName, parentDomainQualifiedName); entity.setAttribute(QUALIFIED_NAME, createQualifiedName(parentDomainQualifiedName)); From f82b907367136e9af368c5bebb33267598ab12f0 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Tue, 30 Apr 2024 11:55:35 +0530 Subject: [PATCH 077/316] moved indexSearch function to PreprocessorUtils --- .../v2/preprocessor/PreProcessorUtils.java | 45 +++++++++++++++++++ .../datamesh/AbstractDomainPreProcessor.java | 42 ++--------------- .../datamesh/DataProductPreProcessor.java | 2 +- .../datamesh/DomainPreProcessor.java | 2 +- .../AbstractGlossaryPreProcessor.java | 35 +-------------- .../glossary/CategoryPreProcessor.java | 2 +- 6 files changed, 53 insertions(+), 75 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index c358e40027c..6b282d17272 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -1,7 +1,10 @@ package org.apache.atlas.repository.store.graph.v2.preprocessor; +import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; @@ -11,13 +14,19 @@ import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.util.NanoIdUtils; import org.apache.atlas.utils.AtlasEntityUtil; +import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.*; + import static org.apache.atlas.repository.Constants.QUERY_COLLECTION_ENTITY_TYPE; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_CATEGORY; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_RESOURCES; +import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; public class PreProcessorUtils { private static final Logger LOG = LoggerFactory.getLogger(PreProcessorUtils.class); @@ -115,4 +124,40 @@ public static String updateQueryResourceAttributes(AtlasTypeRegistry typeRegistr return newCollectionQualifiedName; } + + public static List indexSearchPaginated(Map dsl, Set attributes, EntityDiscoveryService discovery) throws AtlasBaseException { + IndexSearchParams searchParams = new IndexSearchParams(); + List ret = new ArrayList<>(); + + List sortList = new ArrayList<>(0); + sortList.add(mapOf("__timestamp", mapOf("order", "asc"))); + sortList.add(mapOf("__guid", mapOf("order", "asc"))); + dsl.put("sort", sortList); + + int from = 0; + int size = 100; + boolean hasMore = true; + do { + dsl.put("from", from); + dsl.put("size", size); + searchParams.setDsl(dsl); + + if (CollectionUtils.isNotEmpty(attributes)) { + searchParams.setAttributes(attributes); + } + + List headers = discovery.directIndexSearch(searchParams).getEntities(); + + if (CollectionUtils.isNotEmpty(headers)) { + ret.addAll(headers); + } else { + hasMore = false; + } + + from += size; + + } while (hasMore); + + return ret; + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index b0033307a65..4a8423851af 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -24,7 +24,6 @@ import org.apache.atlas.authorize.AtlasPrivilege; import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasObjectId; @@ -44,6 +43,7 @@ import java.util.*; import static org.apache.atlas.repository.Constants.*; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.*; import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_CATEGORY; import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_RESOURCES; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; @@ -68,43 +68,6 @@ public abstract class AbstractDomainPreProcessor implements PreProcessor { } } - public List indexSearchPaginated(Map dsl, String entityType) throws AtlasBaseException { - IndexSearchParams searchParams = new IndexSearchParams(); - List ret = new ArrayList<>(); - - List sortList = new ArrayList<>(0); - sortList.add(mapOf("__timestamp", mapOf("order", "asc"))); - sortList.add(mapOf("__guid", mapOf("order", "asc"))); - dsl.put("sort", sortList); - - int from = 0; - int size = 100; - boolean hasMore = true; - do { - dsl.put("from", from); - dsl.put("size", size); - searchParams.setDsl(dsl); - - if (entityType.equals(POLICY_ENTITY_TYPE)) { - Set attributes = new HashSet<>(Arrays.asList(ATTR_POLICY_RESOURCES, ATTR_POLICY_CATEGORY)); - searchParams.setAttributes(attributes); - } - - List headers = discovery.directIndexSearch(searchParams).getEntities(); - - if (CollectionUtils.isNotEmpty(headers)) { - ret.addAll(headers); - } else { - hasMore = false; - } - - from += size; - - } while (hasMore); - - return ret; - } - protected void isAuthorized(AtlasEntityHeader sourceDomain, AtlasEntityHeader targetDomain) throws AtlasBaseException { // source -> CREATE + UPDATE + DELETE @@ -188,8 +151,9 @@ protected List getPolicy(String resource) throws AtlasBaseExc bool.put("must", mustClauseList); Map dsl = mapOf("query", mapOf("bool", bool)); + Set attributes = new HashSet<>(Arrays.asList(ATTR_POLICY_RESOURCES, ATTR_POLICY_CATEGORY)); - List policies = indexSearchPaginated(dsl, POLICY_ENTITY_TYPE); + List policies = indexSearchPaginated(dsl, attributes, discovery); return policies; } finally { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 4b84a1b450b..0eca39f9ec5 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -230,7 +230,7 @@ private void productExists(String productName, String parentDomainQualifiedName) Map dsl = mapOf("query", mapOf("bool", bool)); - List products = indexSearchPaginated(dsl, DATA_PRODUCT_ENTITY_TYPE); + List products = indexSearchPaginated(dsl, null, this.discovery); if (CollectionUtils.isNotEmpty(products)) { for (AtlasEntityHeader product : products) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index d45dab7fdc7..9d2620de09f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -341,7 +341,7 @@ private void domainExists(String domainName, String parentDomainQualifiedName) t Map dsl = mapOf("query", mapOf("bool", bool)); - List domains = indexSearchPaginated(dsl, DATA_DOMAIN_ENTITY_TYPE); + List domains = indexSearchPaginated(dsl, null, this.discovery); if (CollectionUtils.isNotEmpty(domains)) { for (AtlasEntityHeader domain : domains) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java index 91950f783c5..08c604489c1 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java @@ -58,6 +58,7 @@ import static org.apache.atlas.repository.Constants.ELASTICSEARCH_PAGINATION_SIZE; import static org.apache.atlas.repository.Constants.NAME; import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.indexSearchPaginated; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; import static org.apache.atlas.type.Constants.MEANINGS_PROPERTY_KEY; import static org.apache.atlas.type.Constants.MEANINGS_TEXT_PROPERTY_KEY; @@ -103,7 +104,7 @@ public void termExists(String termName, String glossaryQName) throws AtlasBaseEx Map dsl = mapOf("query", mapOf("bool", mapOf("must", mustClauseList))); - List terms = indexSearchPaginated(dsl); + List terms = indexSearchPaginated(dsl, null, this.discovery); if (CollectionUtils.isNotEmpty(terms)) { ret = terms.stream().map(term -> (String) term.getAttribute(NAME)).anyMatch(name -> termName.equals(name)); @@ -137,38 +138,6 @@ public boolean checkEntityTermAssociation(String termQName) throws AtlasBaseExce return entityHeader != null; } - public List indexSearchPaginated(Map dsl) throws AtlasBaseException { - IndexSearchParams searchParams = new IndexSearchParams(); - List ret = new ArrayList<>(); - - List sortList = new ArrayList<>(0); - sortList.add(mapOf("__timestamp", mapOf("order", "asc"))); - sortList.add(mapOf("__guid", mapOf("order", "asc"))); - dsl.put("sort", sortList); - - int from = 0; - int size = 100; - boolean hasMore = true; - do { - dsl.put("from", from); - dsl.put("size", size); - searchParams.setDsl(dsl); - - List headers = discovery.directIndexSearch(searchParams).getEntities(); - - if (CollectionUtils.isNotEmpty(headers)) { - ret.addAll(headers); - } else { - hasMore = false; - } - - from += size; - - } while (hasMore); - - return ret; - } - public void updateMeaningsAttributesInEntitiesOnTermUpdate(String currentTermName, String updatedTermName, String termQName, String updatedTermQName, String termGuid) throws AtlasBaseException { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java index eb39ff3b1d2..88f72d2f16d 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java @@ -356,7 +356,7 @@ private void categoryExists(String categoryName, String glossaryQualifiedName) t Map dsl = mapOf("query", mapOf("bool", bool)); - List categories = indexSearchPaginated(dsl); + List categories = indexSearchPaginated(dsl, null, this.discovery); if (CollectionUtils.isNotEmpty(categories)) { for (AtlasEntityHeader category : categories) { From 9f26e464622d1c2a7ddd66097f05e32a30c106d6 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Tue, 30 Apr 2024 12:12:42 +0530 Subject: [PATCH 078/316] Add remove user added relationship to control relationship update --- .../contract/ContractPreProcessor.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index 47fc097042c..df3efde85f6 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -81,6 +81,8 @@ private void processUpdateContract(AtlasEntity entity, EntityMutationContext con String contractString = (String) entity.getAttribute(ATTR_CONTRACT); AtlasVertex vertex = context.getVertex(entity.getGuid()); AtlasEntity existingContractEntity = entityRetriever.toAtlasEntity(vertex); + // No update to relationships allowed for the existing contract version + resetAllRelationshipAttributes(entity); if (!isEqualContract(contractString, (String) existingContractEntity.getAttribute(ATTR_CONTRACT))) { // Update the same asset(entity) throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Can't update a specific version of contract"); @@ -128,6 +130,7 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con removeCreatingVertex(context, entity); return; } else if (isEqualContract(contractString, (String) currentVersionEntity.getAttribute(ATTR_CONTRACT))) { + resetAllRelationshipAttributes(entity); // No change in contract, metadata changed updateExistingVersion(context, entity, currentVersionEntity); newVersionNumber = currentVersionNumber; @@ -135,6 +138,7 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con // contract changed (metadata might/not changed). Create new version. newVersionNumber = currentVersionNumber + 1; + resetAllRelationshipAttributes(entity); // Attach previous version via rel entity.setRelationshipAttribute(REL_ATTR_PREVIOUS_VERSION, getAtlasObjectId(currentVersionEntity)); AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(currentVersionEntity.getGuid()); @@ -185,7 +189,7 @@ private void updateExistingVersion(EntityMutationContext context, AtlasEntity en removeCreatingVertex(context, entity); entity.setAttribute(QUALIFIED_NAME, currentVersionEntity.getAttribute(QUALIFIED_NAME)); entity.setGuid(currentVersionEntity.getGuid()); - AtlasVertex vertex = context.getVertex(entity.getGuid()); + AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(entity.getGuid()); AtlasEntityType entityType = ensureEntityType(entity.getTypeName()); context.addUpdated(entity.getGuid(), entity, entityType, vertex); @@ -221,6 +225,18 @@ private void removeCreatingVertex(EntityMutationContext context, AtlasEntity ent graph.removeVertex(context.getVertex(entity.getGuid())); } + private void resetAllRelationshipAttributes(AtlasEntity entity) { + if (entity.getRemoveRelationshipAttributes() != null) { + entity.setRemoveRelationshipAttributes(null); + } + if (entity.getAppendRelationshipAttributes() != null) { + entity.setAppendRelationshipAttributes(null); + } + if (entity.getRelationshipAttributes() != null) { + entity.setRelationshipAttributes(null); + } + } + private boolean syncContractCertificateStatus(AtlasEntity entity, DataContract contract) throws AtlasBaseException { boolean contractSync = false; // Sync certificateStatus From 418a8cf4db73a93d91cf17068538bf9b0f837990 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Tue, 30 Apr 2024 12:17:18 +0530 Subject: [PATCH 079/316] Ignoring null value fields --- .../v2/preprocessor/contract/DataContract.java | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java index ae45bf2353e..dc3cdb466b8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java @@ -21,6 +21,7 @@ @JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) @JsonPropertyOrder({"kind", "status", "template_version", "data_source", "dataset", "type", "description", "owners", "tags", "certificate", "columns"}) public class DataContract { @@ -144,10 +145,6 @@ public void setType(String type) throws AtlasBaseException { } } - public void setDescription(String description) { - this.description = description; - } - public void setOwners(List owners) { this.owners = owners; } @@ -156,10 +153,6 @@ public void setTags(List tags) { this.tags = tags; } - public void setCertificate(String certificate) { - this.certificate = certificate; - } - public void setColumns(List columns) { this.columns = columns; } @@ -169,13 +162,13 @@ public void setUnknownFields(String key, Object value) { unknownFields.put(key, value); } - private boolean isSemVer(String version) { Matcher matcher = versionPattern.matcher(version); return matcher.matches(); } @JsonIgnoreProperties(ignoreUnknown = true) + @JsonInclude(JsonInclude.Include.NON_NULL) @JsonPropertyOrder({"name"}) public static final class BusinessTag { @NotNull @@ -194,6 +187,7 @@ public Map getUnknownFields() { } @JsonIgnoreProperties(ignoreUnknown = true) + @JsonInclude(JsonInclude.Include.NON_NULL) @JsonPropertyOrder({"name", "description", "data_type"}) public static final class Column { @NotNull @@ -201,8 +195,6 @@ public static final class Column { public String description; - public boolean is_primary; - public String data_type; private final Map unknownFields = new HashMap<>(); From fb0ee62b5773fc86e5478985c0ddb6b5d8c8780f Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Tue, 30 Apr 2024 18:03:20 +0530 Subject: [PATCH 080/316] Upgrade validator version to avoid vulnerability --- repository/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/pom.xml b/repository/pom.xml index 64cb9c0d48c..a2a1a4198f0 100755 --- a/repository/pom.xml +++ b/repository/pom.xml @@ -325,7 +325,7 @@ org.hibernate hibernate-validator - 4.3.0.Final + 4.3.2.Final From 6777a38a4e6213c0c86a9b0ebd85a4c0ae3b4508 Mon Sep 17 00:00:00 2001 From: Rajat Movaliya Date: Tue, 30 Apr 2024 18:20:14 +0530 Subject: [PATCH 081/316] add depth in list lineage api --- .../model/instance/AtlasEntityHeader.java | 9 +++++++++ .../atlas/discovery/EntityLineageService.java | 19 ++++++++++++++----- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java index be2819d7f20..964b688ced5 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java @@ -57,6 +57,7 @@ public class AtlasEntityHeader extends AtlasStruct implements Serializable { private static final long serialVersionUID = 1L; private String guid = null; + private Integer depth = null; private AtlasEntity.Status status = AtlasEntity.Status.ACTIVE; private String displayText = null; private List classificationNames = null; @@ -154,6 +155,14 @@ public void setGuid(String guid) { this.guid = guid; } + public Integer getDepth() { + return depth; + } + + public void setDepth(Integer depth) { + this.depth = depth; + } + public AtlasEntity.Status getStatus() { return status; } diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index 7c69099e746..0517825d727 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -442,13 +442,20 @@ private void traverseEdgesUsingBFS(String baseGuid, AtlasLineageListContext line visitedVertices.add(baseGuid); Set skippedVertices = new HashSet<>(); Queue traversalQueue = new LinkedList<>(); + boolean isDataset; AtlasVertex baseVertex = AtlasGraphUtilsV2.findByGuid(this.graph, baseGuid); - enqueueNeighbours(baseVertex, validateEntityTypeAndCheckIfDataSet(baseGuid), lineageListContext, traversalQueue, visitedVertices, skippedVertices); + isDataset = validateEntityTypeAndCheckIfDataSet(baseGuid); + enqueueNeighbours(baseVertex, isDataset, lineageListContext, traversalQueue, visitedVertices, skippedVertices); int currentDepth = 0; + int level = 0; while (!traversalQueue.isEmpty() && !lineageListContext.isEntityLimitReached() && currentDepth < lineageListContext.getDepth()) { currentDepth++; + + if (!isDataset) + level++; + int entitiesInCurrentDepth = traversalQueue.size(); for (int i = 0; i < entitiesInCurrentDepth; i++) { if (lineageListContext.isEntityLimitReached()) @@ -459,7 +466,7 @@ private void traverseEdgesUsingBFS(String baseGuid, AtlasLineageListContext line if (Objects.isNull(currentVertex)) throw new AtlasBaseException("Found null vertex during lineage graph traversal for guid: " + currentGUID); - boolean isDataset = validateEntityTypeAndCheckIfDataSet(currentGUID); + isDataset = validateEntityTypeAndCheckIfDataSet(currentGUID); if (!lineageListContext.evaluateVertexFilter(currentVertex)) { enqueueNeighbours(currentVertex, isDataset, lineageListContext, traversalQueue, visitedVertices, skippedVertices); continue; @@ -471,7 +478,7 @@ private void traverseEdgesUsingBFS(String baseGuid, AtlasLineageListContext line } lineageListContext.incrementEntityCount(); - appendToResult(currentVertex, lineageListContext, ret); + appendToResult(currentVertex, lineageListContext, ret, level); enqueueNeighbours(currentVertex, isDataset, lineageListContext, traversalQueue, visitedVertices, skippedVertices); if (isLastEntityInLastDepth(lineageListContext.getDepth(), currentDepth, entitiesInCurrentDepth, i)) { ret.setHasMore(false); @@ -518,8 +525,10 @@ private void enqueueNeighbours(AtlasVertex currentVertex, boolean isDataset, Atl } } - private void appendToResult(AtlasVertex currentVertex, AtlasLineageListContext lineageListContext, AtlasLineageListInfo ret) throws AtlasBaseException { - ret.getEntities().add(entityRetriever.toAtlasEntityHeader(currentVertex, lineageListContext.getAttributes())); + private void appendToResult(AtlasVertex currentVertex, AtlasLineageListContext lineageListContext, AtlasLineageListInfo ret, Integer level) throws AtlasBaseException { + AtlasEntityHeader entity = entityRetriever.toAtlasEntityHeader(currentVertex, lineageListContext.getAttributes()); + entity.setDepth(level); + ret.getEntities().add(entity); } private static void addEntitiesToCache(AtlasVertex vertex) { From c6b4c5d6b105f673af533a6a998c984d9231b1cb Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Tue, 30 Apr 2024 23:45:00 +0530 Subject: [PATCH 082/316] feat: add perf metrics --- graphdb/janus/pom.xml | 4 ++ .../janus/AtlasElasticsearchQuery.java | 4 ++ .../graphdb/janus/SearchContextCache.java | 58 ++++++++++++------- 3 files changed, 46 insertions(+), 20 deletions(-) diff --git a/graphdb/janus/pom.xml b/graphdb/janus/pom.xml index 5daef76b4ee..75c9079eee1 100644 --- a/graphdb/janus/pom.xml +++ b/graphdb/janus/pom.xml @@ -282,6 +282,10 @@ mockito-all test + + org.apache.atlas + atlas-server-api + diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java index a4e6bde67b6..127439dc307 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java @@ -19,12 +19,14 @@ import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.RequestContext; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.discovery.SearchParams; import org.apache.atlas.repository.graphdb.AtlasIndexQuery; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.graphdb.DirectIndexQueryResult; import org.apache.atlas.type.AtlasType; +import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.NotImplementedException; import org.apache.commons.lang.StringUtils; @@ -171,6 +173,7 @@ private Map runQueryWithLowLevelClient(String query) throws Atla } private DirectIndexQueryResult performAsyncDirectIndexQuery(SearchParams searchParams) throws AtlasBaseException, IOException { + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("performAsyncDirectIndexQuery"); DirectIndexQueryResult result = null; boolean contextIdExists = StringUtils.isNotEmpty(searchParams.getSearchContextId()) && searchParams.getSearchContextSequenceNo() != null; try { @@ -222,6 +225,7 @@ private DirectIndexQueryResult performAsyncDirectIndexQuery(SearchParams searchP LOG.error("Failed to remove the search context from the cache {}", e.getMessage()); } } + RequestContext.get().endMetricRecord(metric); } return result; } diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/SearchContextCache.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/SearchContextCache.java index 6e1559010e3..fc627ef51f6 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/SearchContextCache.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/SearchContextCache.java @@ -2,8 +2,10 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; +import org.apache.atlas.RequestContext; import org.apache.atlas.service.redis.AbstractRedisService; import org.apache.atlas.service.redis.RedisService; +import org.apache.atlas.utils.AtlasPerfMetrics; import org.slf4j.Logger; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; @@ -25,36 +27,52 @@ public SearchContextCache(@Qualifier("redisServiceImpl") RedisService redisServi public static void put(String key, Integer sequence, String esAsyncId) { - // Build the string in format `sequence/esAsyncId` and store it in redis - String val = sequence + "/" + esAsyncId; - redisService.putValue(key, val); + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("putInCache"); + try { + // Build the string in format `sequence/esAsyncId` and store it in redis + String val = sequence + "/" + esAsyncId; + redisService.putValue(key, val); + } finally { + RequestContext.get().endMetricRecord(metric); + } } public static String get(String key){ return redisService.getValue(key); } public static String getESAsyncSearchIdFromContextCache(String key, Integer sequence){ - //Get the context cache for the given key - String contextCache = get(key); - if(contextCache == null || sequence == null){ + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("getESAsyncSearchIdFromContextCache"); + try { + //Get the context cache for the given key + String contextCache = get(key); + if(contextCache == null || sequence == null){ + return null; + } + // Split the context cache to get the sequence and ESAsyncId + String[] contextCacheSplit = contextCache.split("/"); + if(contextCacheSplit.length != 2){ + return null; + } + int seq = Integer.parseInt(contextCacheSplit[0]); + if(sequence > seq){ + return contextCacheSplit[1]; + } else if (sequence < seq) { + return INVALID_SEQUENCE; + } return null; + } finally { + RequestContext.get().endMetricRecord(metric); } - // Split the context cache to get the sequence and ESAsyncId - String[] contextCacheSplit = contextCache.split("/"); - if(contextCacheSplit.length != 2){ - return null; - } - int seq = Integer.parseInt(contextCacheSplit[0]); - if(sequence > seq){ - return contextCacheSplit[1]; - } else if (sequence < seq) { - return INVALID_SEQUENCE; - } - return null; - } + } public static void remove(String key) { - redisService.removeValue(key); + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("removeFromCache"); + try { + redisService.removeValue(key); + } finally { + RequestContext.get().endMetricRecord(metric); + } + } } From 7738fd9e62c6e3c73fdf4c186b81b38ba45bf81a Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Wed, 1 May 2024 00:09:19 +0530 Subject: [PATCH 083/316] feat: add perf for async search check --- .../janus/AtlasElasticsearchQuery.java | 25 +++++++++++-------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java index 127439dc307..51e18379751 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java @@ -240,16 +240,21 @@ private DirectIndexQueryResult performAsyncDirectIndexQuery(SearchParams searchP * We also need to check if the search ID exists and delete if necessary */ private void processRequestWithSameSearchContextId(SearchParams searchParams) { - // Extract search context ID and sequence number - String currentSearchContextId = searchParams.getSearchContextId(); - Integer currentSequenceNumber = searchParams.getSearchContextSequenceNo(); - // Get the search ID from the cache if sequence number is greater than the current sequence number - String previousESSearchId = SearchContextCache.getESAsyncSearchIdFromContextCache(currentSearchContextId, currentSequenceNumber); - - if (StringUtils.isNotEmpty(previousESSearchId)) { - LOG.debug("Deleting the previous async search response with ID {}", previousESSearchId); - // If the search ID exists, then we need to delete the search context - deleteAsyncSearchResponse(previousESSearchId); + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("processRequestWithSameSearchContextId"); + try { + // Extract search context ID and sequence number + String currentSearchContextId = searchParams.getSearchContextId(); + Integer currentSequenceNumber = searchParams.getSearchContextSequenceNo(); + // Get the search ID from the cache if sequence number is greater than the current sequence number + String previousESSearchId = SearchContextCache.getESAsyncSearchIdFromContextCache(currentSearchContextId, currentSequenceNumber); + + if (StringUtils.isNotEmpty(previousESSearchId)) { + LOG.debug("Deleting the previous async search response with ID {}", previousESSearchId); + // If the search ID exists, then we need to delete the search context + deleteAsyncSearchResponse(previousESSearchId); + } + } finally { + RequestContext.get().endMetricRecord(metric); } } From 651792d15e6299128e5ea8b00fe749be23de8bbf Mon Sep 17 00:00:00 2001 From: Rajat Movaliya Date: Wed, 1 May 2024 13:07:35 +0530 Subject: [PATCH 084/316] correct the depth for process base node --- .../atlas/model/instance/AtlasEntityHeader.java | 4 ++-- .../atlas/discovery/EntityLineageService.java | 14 +++++++------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java index 964b688ced5..883a3014a33 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java @@ -155,11 +155,11 @@ public void setGuid(String guid) { this.guid = guid; } - public Integer getDepth() { + public int getDepth() { return depth; } - public void setDepth(Integer depth) { + public void setDepth(int depth) { this.depth = depth; } diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index 0517825d727..75b5c6eed50 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -442,18 +442,18 @@ private void traverseEdgesUsingBFS(String baseGuid, AtlasLineageListContext line visitedVertices.add(baseGuid); Set skippedVertices = new HashSet<>(); Queue traversalQueue = new LinkedList<>(); - boolean isDataset; AtlasVertex baseVertex = AtlasGraphUtilsV2.findByGuid(this.graph, baseGuid); - isDataset = validateEntityTypeAndCheckIfDataSet(baseGuid); - enqueueNeighbours(baseVertex, isDataset, lineageListContext, traversalQueue, visitedVertices, skippedVertices); + boolean isBaseNodeDataset = validateEntityTypeAndCheckIfDataSet(baseGuid); + enqueueNeighbours(baseVertex, isBaseNodeDataset, lineageListContext, traversalQueue, visitedVertices, skippedVertices); int currentDepth = 0; - int level = 0; + int level = isBaseNodeDataset? 0: 1; while (!traversalQueue.isEmpty() && !lineageListContext.isEntityLimitReached() && currentDepth < lineageListContext.getDepth()) { currentDepth++; - if (!isDataset) + // update level at every alternate depth + if ((isBaseNodeDataset && currentDepth % 2 != 0) || (!isBaseNodeDataset && currentDepth % 2 == 0)) level++; int entitiesInCurrentDepth = traversalQueue.size(); @@ -466,7 +466,7 @@ private void traverseEdgesUsingBFS(String baseGuid, AtlasLineageListContext line if (Objects.isNull(currentVertex)) throw new AtlasBaseException("Found null vertex during lineage graph traversal for guid: " + currentGUID); - isDataset = validateEntityTypeAndCheckIfDataSet(currentGUID); + boolean isDataset = validateEntityTypeAndCheckIfDataSet(currentGUID); if (!lineageListContext.evaluateVertexFilter(currentVertex)) { enqueueNeighbours(currentVertex, isDataset, lineageListContext, traversalQueue, visitedVertices, skippedVertices); continue; @@ -525,7 +525,7 @@ private void enqueueNeighbours(AtlasVertex currentVertex, boolean isDataset, Atl } } - private void appendToResult(AtlasVertex currentVertex, AtlasLineageListContext lineageListContext, AtlasLineageListInfo ret, Integer level) throws AtlasBaseException { + private void appendToResult(AtlasVertex currentVertex, AtlasLineageListContext lineageListContext, AtlasLineageListInfo ret, int level) throws AtlasBaseException { AtlasEntityHeader entity = entityRetriever.toAtlasEntityHeader(currentVertex, lineageListContext.getAttributes()); entity.setDepth(level); ret.getEntities().add(entity); From 34dc156ec815530cef6bfbfbfe29ce2232ec3588 Mon Sep 17 00:00:00 2001 From: Rajat Movaliya Date: Wed, 1 May 2024 14:25:50 +0530 Subject: [PATCH 085/316] rename variable --- .../apache/atlas/discovery/EntityLineageService.java | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index 75b5c6eed50..e7f3a10501b 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -447,14 +447,14 @@ private void traverseEdgesUsingBFS(String baseGuid, AtlasLineageListContext line boolean isBaseNodeDataset = validateEntityTypeAndCheckIfDataSet(baseGuid); enqueueNeighbours(baseVertex, isBaseNodeDataset, lineageListContext, traversalQueue, visitedVertices, skippedVertices); int currentDepth = 0; - int level = isBaseNodeDataset? 0: 1; + int currentLevel = isBaseNodeDataset? 0: 1; while (!traversalQueue.isEmpty() && !lineageListContext.isEntityLimitReached() && currentDepth < lineageListContext.getDepth()) { currentDepth++; // update level at every alternate depth if ((isBaseNodeDataset && currentDepth % 2 != 0) || (!isBaseNodeDataset && currentDepth % 2 == 0)) - level++; + currentLevel++; int entitiesInCurrentDepth = traversalQueue.size(); for (int i = 0; i < entitiesInCurrentDepth; i++) { @@ -478,7 +478,7 @@ private void traverseEdgesUsingBFS(String baseGuid, AtlasLineageListContext line } lineageListContext.incrementEntityCount(); - appendToResult(currentVertex, lineageListContext, ret, level); + appendToResult(currentVertex, lineageListContext, ret, currentLevel); enqueueNeighbours(currentVertex, isDataset, lineageListContext, traversalQueue, visitedVertices, skippedVertices); if (isLastEntityInLastDepth(lineageListContext.getDepth(), currentDepth, entitiesInCurrentDepth, i)) { ret.setHasMore(false); @@ -525,9 +525,9 @@ private void enqueueNeighbours(AtlasVertex currentVertex, boolean isDataset, Atl } } - private void appendToResult(AtlasVertex currentVertex, AtlasLineageListContext lineageListContext, AtlasLineageListInfo ret, int level) throws AtlasBaseException { + private void appendToResult(AtlasVertex currentVertex, AtlasLineageListContext lineageListContext, AtlasLineageListInfo ret, int currentLevel) throws AtlasBaseException { AtlasEntityHeader entity = entityRetriever.toAtlasEntityHeader(currentVertex, lineageListContext.getAttributes()); - entity.setDepth(level); + entity.setDepth(currentLevel); ret.getEntities().add(entity); } From a41e3c08ffa6488ae251fc416f51b9fc4294b528 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Wed, 1 May 2024 14:48:41 +0530 Subject: [PATCH 086/316] Revert "feat: Search cancellation from Metastore with global context cache using Redis" --- .../service/redis/AbstractRedisService.java | 26 +----- .../service/redis/NoRedisServiceImpl.java | 15 ---- .../atlas/service/redis/RedisService.java | 6 -- .../service/redis/RedisServiceLocalImpl.java | 15 ---- graphdb/janus/pom.xml | 4 - .../janus/AtlasElasticsearchQuery.java | 56 ++++--------- .../graphdb/janus/SearchContextCache.java | 81 ++++++------------- 7 files changed, 42 insertions(+), 161 deletions(-) diff --git a/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java b/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java index 5bd2785ba04..0ae20a60cc4 100644 --- a/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java +++ b/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java @@ -71,25 +71,6 @@ public void releaseDistributedLock(String key) { } } - @Override - public String getValue(String key) { - // If value doesn't exist, return null else return the value - return (String) redisClient.getBucket(convertToNamespace(key)).get(); - } - - @Override - public String putValue(String key, String value) { - // Put the value in the redis cache with TTL - redisClient.getBucket(convertToNamespace(key)).set(value, 30, TimeUnit.SECONDS); - return value; - } - - @Override - public void removeValue(String key) { - // Remove the value from the redis cache - redisClient.getBucket(convertToNamespace(key)).delete(); - } - private String getHostAddress() throws UnknownHostException { return InetAddress.getLocalHost().getHostAddress(); } @@ -104,11 +85,6 @@ private Config initAtlasConfig() throws AtlasException { return redisConfig; } - private String convertToNamespace(String key){ - // Append key with namespace :atlas - return "atlas:"+key; - } - Config getLocalConfig() throws AtlasException { Config config = initAtlasConfig(); config.useSingleServer() @@ -118,7 +94,7 @@ Config getLocalConfig() throws AtlasException { return config; } - protected Config getProdConfig() throws AtlasException { + Config getProdConfig() throws AtlasException { Config config = initAtlasConfig(); config.useSentinelServers() .setClientName(ATLAS_METASTORE_SERVICE) diff --git a/common/src/main/java/org/apache/atlas/service/redis/NoRedisServiceImpl.java b/common/src/main/java/org/apache/atlas/service/redis/NoRedisServiceImpl.java index 5bc089d9557..96a8fadc99f 100644 --- a/common/src/main/java/org/apache/atlas/service/redis/NoRedisServiceImpl.java +++ b/common/src/main/java/org/apache/atlas/service/redis/NoRedisServiceImpl.java @@ -29,21 +29,6 @@ public void releaseDistributedLock(String key) { //do nothing } - @Override - public String getValue(String key) { - return null; - } - - @Override - public String putValue(String key, String value) { - return null; - } - - @Override - public void removeValue(String key) { - - } - @Override public Logger getLogger() { return LOG; diff --git a/common/src/main/java/org/apache/atlas/service/redis/RedisService.java b/common/src/main/java/org/apache/atlas/service/redis/RedisService.java index 6cb2f04ada3..1475f93e832 100644 --- a/common/src/main/java/org/apache/atlas/service/redis/RedisService.java +++ b/common/src/main/java/org/apache/atlas/service/redis/RedisService.java @@ -8,12 +8,6 @@ public interface RedisService { void releaseDistributedLock(String key); - String getValue(String key); - - String putValue(String key, String value); - - void removeValue(String key); - Logger getLogger(); } diff --git a/common/src/main/java/org/apache/atlas/service/redis/RedisServiceLocalImpl.java b/common/src/main/java/org/apache/atlas/service/redis/RedisServiceLocalImpl.java index 1ddad0f3e80..2eb774920ef 100644 --- a/common/src/main/java/org/apache/atlas/service/redis/RedisServiceLocalImpl.java +++ b/common/src/main/java/org/apache/atlas/service/redis/RedisServiceLocalImpl.java @@ -21,21 +21,6 @@ public void init() throws AtlasException { LOG.info("Local redis client created successfully."); } - @Override - public String getValue(String key) { - return null; - } - - @Override - public String putValue(String key, String value) { - return null; - } - - @Override - public void removeValue(String key) { - - } - @Override public Logger getLogger() { return LOG; diff --git a/graphdb/janus/pom.xml b/graphdb/janus/pom.xml index 75c9079eee1..5daef76b4ee 100644 --- a/graphdb/janus/pom.xml +++ b/graphdb/janus/pom.xml @@ -282,10 +282,6 @@ mockito-all test - - org.apache.atlas - atlas-server-api - diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java index 51e18379751..9d2e2489e8a 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java @@ -19,14 +19,12 @@ import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasErrorCode; -import org.apache.atlas.RequestContext; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.discovery.SearchParams; import org.apache.atlas.repository.graphdb.AtlasIndexQuery; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.graphdb.DirectIndexQueryResult; import org.apache.atlas.type.AtlasType; -import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.NotImplementedException; import org.apache.commons.lang.StringUtils; @@ -173,17 +171,11 @@ private Map runQueryWithLowLevelClient(String query) throws Atla } private DirectIndexQueryResult performAsyncDirectIndexQuery(SearchParams searchParams) throws AtlasBaseException, IOException { - AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("performAsyncDirectIndexQuery"); DirectIndexQueryResult result = null; - boolean contextIdExists = StringUtils.isNotEmpty(searchParams.getSearchContextId()) && searchParams.getSearchContextSequenceNo() != null; try { - if(contextIdExists) { + if(StringUtils.isNotEmpty(searchParams.getSearchContextId()) && searchParams.getSearchContextSequenceNo() != null) { // If the search context id and greater sequence no is present, then we need to delete the previous search context async - try { - processRequestWithSameSearchContextId(searchParams); - } catch (Exception e) { - LOG.error("Failed to process the request with the same search context ID {}", e.getMessage()); - } + processRequestWithSameSearchContextId(searchParams); } AsyncQueryResult response = submitAsyncSearch(searchParams, false).get(); @@ -197,12 +189,8 @@ private DirectIndexQueryResult performAsyncDirectIndexQuery(SearchParams searchP String esSearchId = response.getId(); String searchContextId = searchParams.getSearchContextId(); Integer searchContextSequenceNo = searchParams.getSearchContextSequenceNo(); - if (contextIdExists) { - try { - SearchContextCache.put(searchContextId, searchContextSequenceNo, esSearchId); - } catch (Exception e) { - LOG.error("Failed to update the search context cache {}", e.getMessage()); - } + if (StringUtils.isNotEmpty(searchContextId) && searchContextSequenceNo != null) { + SearchContextCache.put(searchContextId, searchContextSequenceNo, esSearchId); } response = getAsyncSearchResponse(searchParams, esSearchId).get(); if (response == null) { @@ -216,16 +204,6 @@ private DirectIndexQueryResult performAsyncDirectIndexQuery(SearchParams searchP }catch (Exception e) { LOG.error("Failed to execute direct query on ES {}", e.getMessage()); throw new AtlasBaseException(AtlasErrorCode.INDEX_SEARCH_FAILED, e.getMessage()); - } finally { - if (contextIdExists) { - // If the search context id is present, then we need to remove the search context from the cache - try { - SearchContextCache.remove(searchParams.getSearchContextId()); - } catch (Exception e) { - LOG.error("Failed to remove the search context from the cache {}", e.getMessage()); - } - } - RequestContext.get().endMetricRecord(metric); } return result; } @@ -240,21 +218,17 @@ private DirectIndexQueryResult performAsyncDirectIndexQuery(SearchParams searchP * We also need to check if the search ID exists and delete if necessary */ private void processRequestWithSameSearchContextId(SearchParams searchParams) { - AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("processRequestWithSameSearchContextId"); - try { - // Extract search context ID and sequence number - String currentSearchContextId = searchParams.getSearchContextId(); - Integer currentSequenceNumber = searchParams.getSearchContextSequenceNo(); - // Get the search ID from the cache if sequence number is greater than the current sequence number - String previousESSearchId = SearchContextCache.getESAsyncSearchIdFromContextCache(currentSearchContextId, currentSequenceNumber); - - if (StringUtils.isNotEmpty(previousESSearchId)) { - LOG.debug("Deleting the previous async search response with ID {}", previousESSearchId); - // If the search ID exists, then we need to delete the search context - deleteAsyncSearchResponse(previousESSearchId); - } - } finally { - RequestContext.get().endMetricRecord(metric); + // Extract search context ID and sequence number + String currentSearchContextId = searchParams.getSearchContextId(); + Integer currentSequenceNumber = searchParams.getSearchContextSequenceNo(); + // Get the search ID from the cache if sequence number is greater than the current sequence number + String previousESSearchId = SearchContextCache.getESAsyncSearchIdFromContextCache(currentSearchContextId, currentSequenceNumber); + + if (StringUtils.isNotEmpty(previousESSearchId)) { + LOG.debug("Deleting the previous async search response with ID {}", previousESSearchId); + // If the search ID exists, then we need to delete the search context + deleteAsyncSearchResponse(previousESSearchId); + SearchContextCache.remove(currentSearchContextId); } } diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/SearchContextCache.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/SearchContextCache.java index fc627ef51f6..1780e5d1f90 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/SearchContextCache.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/SearchContextCache.java @@ -2,77 +2,48 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; -import org.apache.atlas.RequestContext; -import org.apache.atlas.service.redis.AbstractRedisService; -import org.apache.atlas.service.redis.RedisService; -import org.apache.atlas.utils.AtlasPerfMetrics; -import org.slf4j.Logger; -import org.springframework.beans.factory.annotation.Qualifier; -import org.springframework.stereotype.Component; import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; -@Component public class SearchContextCache { - private static RedisService redisService = null; - - public static final String INVALID_SEQUENCE = "invalid_sequence"; - - - public SearchContextCache(@Qualifier("redisServiceImpl") RedisService redisService) { - SearchContextCache.redisService = redisService; - } - + private static final Cache> searchContextCache = CacheBuilder.newBuilder() + .maximumSize(200) + .expireAfterWrite(30, TimeUnit.SECONDS) + .build(); public static void put(String key, Integer sequence, String esAsyncId) { - AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("putInCache"); - try { - // Build the string in format `sequence/esAsyncId` and store it in redis - String val = sequence + "/" + esAsyncId; - redisService.putValue(key, val); - } finally { - RequestContext.get().endMetricRecord(metric); - } + HashMap entry = new HashMap<>(); + entry.put(sequence, esAsyncId); + searchContextCache.put(key, entry); } - public static String get(String key){ - return redisService.getValue(key); + public static HashMap get(String key){ + return searchContextCache.getIfPresent(key); } public static String getESAsyncSearchIdFromContextCache(String key, Integer sequence){ - AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("getESAsyncSearchIdFromContextCache"); - try { - //Get the context cache for the given key - String contextCache = get(key); - if(contextCache == null || sequence == null){ - return null; - } - // Split the context cache to get the sequence and ESAsyncId - String[] contextCacheSplit = contextCache.split("/"); - if(contextCacheSplit.length != 2){ - return null; - } - int seq = Integer.parseInt(contextCacheSplit[0]); - if(sequence > seq){ - return contextCacheSplit[1]; - } else if (sequence < seq) { - return INVALID_SEQUENCE; - } + //Get the context cache for the given key + HashMap contextCache = get(key); + if(contextCache == null || sequence == null){ return null; - } finally { - RequestContext.get().endMetricRecord(metric); } - + //Find the highest sequence number + int maxStoredSequence = 0; + for (Integer seq : contextCache.keySet()) { + if (seq > maxStoredSequence) { + maxStoredSequence = seq; + } + } + //If the given sequence is greater than the max stored sequence, return the ESAsyncId else return null + return sequence > maxStoredSequence ? contextCache.getOrDefault(maxStoredSequence, null) : null; } + public static void remove(String key) { - AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("removeFromCache"); - try { - redisService.removeValue(key); - } finally { - RequestContext.get().endMetricRecord(metric); - } + searchContextCache.invalidate(key); + } + public static void clear() { + searchContextCache.cleanUp(); } } - From a34d552af23dfc0d51933f051d281a69833b20e6 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Fri, 19 Apr 2024 13:50:16 +0530 Subject: [PATCH 087/316] Removed Column Lineage --- .../main/java/org/apache/atlas/repository/Constants.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index e10796992d3..fe46e5a4606 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -378,7 +378,6 @@ public enum SupportedFileExtensions { XLSX, XLS, CSV } public static final String CATALOG_PROCESS_INPUT_RELATIONSHIP_LABEL = "__Process.inputs"; public static final String CATALOG_PROCESS_OUTPUT_RELATIONSHIP_LABEL = "__Process.outputs"; - public static final String COLUMN_LINEAGE_RELATIONSHIP_LABEL = "__Process.columnProcesses"; public static final String CLASSIFICATION_PROPAGATION_MODE_DEFAULT ="DEFAULT"; public static final String CLASSIFICATION_PROPAGATION_MODE_RESTRICT_LINEAGE ="RESTRICT_LINEAGE"; @@ -388,14 +387,12 @@ public enum SupportedFileExtensions { XLSX, XLS, CSV } public static final HashMap> CLASSIFICATION_PROPAGATION_MODE_LABELS_MAP = new HashMap>(){{ put(CLASSIFICATION_PROPAGATION_MODE_RESTRICT_LINEAGE, new ArrayList<>( Arrays.asList(CATALOG_PROCESS_INPUT_RELATIONSHIP_LABEL, - CATALOG_PROCESS_OUTPUT_RELATIONSHIP_LABEL, - COLUMN_LINEAGE_RELATIONSHIP_LABEL + CATALOG_PROCESS_OUTPUT_RELATIONSHIP_LABEL ))); put(CLASSIFICATION_PROPAGATION_MODE_DEFAULT, null); put(CLASSIFICATION_PROPAGATION_MODE_RESTRICT_HIERARCHY, new ArrayList<>( Arrays.asList(CATALOG_PROCESS_INPUT_RELATIONSHIP_LABEL, - CATALOG_PROCESS_OUTPUT_RELATIONSHIP_LABEL, - COLUMN_LINEAGE_RELATIONSHIP_LABEL + CATALOG_PROCESS_OUTPUT_RELATIONSHIP_LABEL ))); }}; From 0ec7b748cced0b17a25ff6781e819e97fb5bbfb1 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Tue, 30 Apr 2024 13:44:30 +0530 Subject: [PATCH 088/316] Fixed the updatedTagPropagation logic that was running the compute propagatedEntityVertices logic even when it should not have --- .../repository/store/graph/v2/EntityGraphMapper.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index a95ca20f0ca..4f7ceb05884 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -137,8 +137,7 @@ import static org.apache.atlas.repository.graph.GraphHelper.getPropagatableClassifications; import static org.apache.atlas.repository.graph.GraphHelper.getClassificationEntityGuid; import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.*; -import static org.apache.atlas.repository.store.graph.v2.tasks.ClassificationPropagateTaskFactory.CLASSIFICATION_PROPAGATION_ADD; -import static org.apache.atlas.repository.store.graph.v2.tasks.ClassificationPropagateTaskFactory.CLASSIFICATION_PROPAGATION_DELETE; +import static org.apache.atlas.repository.store.graph.v2.tasks.ClassificationPropagateTaskFactory.*; import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.IN; import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.OUT; import static org.apache.atlas.type.Constants.PENDING_TASKS_PROPERTY_KEY; @@ -3587,11 +3586,14 @@ public void updateClassifications(EntityMutationContext context, String guid, Li taskManagement != null && DEFERRED_ACTION_ENABLED) { String propagationType = CLASSIFICATION_PROPAGATION_ADD; - if (removePropagation || !updatedTagPropagation) - { + if(currentRestrictPropagationThroughLineage != updatedRestrictPropagationThroughLineage || currentRestrictPropagationThroughHierarchy != updatedRestrictPropagationThroughHierarchy){ + propagationType = CLASSIFICATION_REFRESH_PROPAGATION; + } + if (removePropagation || !updatedTagPropagation) { propagationType = CLASSIFICATION_PROPAGATION_DELETE; } createAndQueueTask(propagationType, entityVertex, classificationVertex.getIdForDisplay(), currentRestrictPropagationThroughLineage,currentRestrictPropagationThroughHierarchy); + updatedTagPropagation = null; } // compute propagatedEntityVertices once and use it for subsequent iterations and notifications From 160e7bd8a8b1e382b5c9f5bca29daa0a92e5d6dc Mon Sep 17 00:00:00 2001 From: hr2904 Date: Tue, 30 Apr 2024 16:35:45 +0530 Subject: [PATCH 089/316] edge case missed Incase all options are true error should be thrown --- .../repository/store/graph/v2/EntityGraphMapper.java | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 4f7ceb05884..a5bf1de5d06 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -3579,7 +3579,7 @@ public void updateClassifications(EntityMutationContext context, String guid, Li Boolean updatedRestrictPropagationThroughLineage = classification.getRestrictPropagationThroughLineage(); Boolean currentRestrictPropagationThroughHierarchy = currentClassification.getRestrictPropagationThroughHierarchy(); Boolean updatedRestrictPropagationThroughHierarchy = classification.getRestrictPropagationThroughHierarchy(); - + String propagationMode = entityRetriever.determinePropagationMode(updatedRestrictPropagationThroughLineage, updatedRestrictPropagationThroughHierarchy); if ((!Objects.equals(updatedRemovePropagations, currentRemovePropagations) || !Objects.equals(currentTagPropagation, updatedTagPropagation) || !Objects.equals(currentRestrictPropagationThroughLineage, updatedRestrictPropagationThroughLineage)) && @@ -3607,11 +3607,7 @@ public void updateClassifications(EntityMutationContext context, String guid, Li deleteDelegate.getHandler().removeTagPropagation(classificationVertex); } if (CollectionUtils.isEmpty(entitiesToPropagateTo)) { - String propagationMode; - if (updatedRemovePropagations !=null) { - propagationMode = entityRetriever.determinePropagationMode(updatedRestrictPropagationThroughLineage, updatedRestrictPropagationThroughHierarchy); - } - else{ + if (updatedRemovePropagations ==null) { propagationMode = CLASSIFICATION_PROPAGATION_MODE_DEFAULT; } Boolean toExclude = propagationMode == CLASSIFICATION_VERTEX_RESTRICT_PROPAGATE_THROUGH_LINEAGE ? true : false; From 8ca65e232b5e24adb618086f88fda0c3d0a08590 Mon Sep 17 00:00:00 2001 From: Rajat Movaliya Date: Wed, 1 May 2024 16:02:22 +0530 Subject: [PATCH 090/316] resolve conflict --- .../atlas/model/instance/AtlasEntityHeader.java | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java index 883a3014a33..f5f060349dd 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java @@ -57,7 +57,6 @@ public class AtlasEntityHeader extends AtlasStruct implements Serializable { private static final long serialVersionUID = 1L; private String guid = null; - private Integer depth = null; private AtlasEntity.Status status = AtlasEntity.Status.ACTIVE; private String displayText = null; private List classificationNames = null; @@ -72,8 +71,10 @@ public class AtlasEntityHeader extends AtlasStruct implements Serializable { private Date createTime = null; private Date updateTime = null; private String deleteHandler = null; + private Integer depth = null; private Map collapse = null; + public AtlasEntityHeader() { this(null, null); } @@ -155,13 +156,13 @@ public void setGuid(String guid) { this.guid = guid; } - public int getDepth() { - return depth; - } + public Integer getDepth() { + return depth; + } - public void setDepth(int depth) { - this.depth = depth; - } + public void setDepth(Integer depth) { + this.depth = depth; + } public AtlasEntity.Status getStatus() { return status; From 29adef55734966e79cb30ed99cc88c566c0cc658 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Wed, 1 May 2024 16:05:04 +0530 Subject: [PATCH 091/316] feat: run Redis ops async and reduce wait time for search cache --- .../service/redis/AbstractRedisService.java | 28 +++++++++++++++++-- .../atlas/service/redis/RedisServiceImpl.java | 2 ++ .../service/redis/RedisServiceLocalImpl.java | 1 + .../janus/AtlasElasticsearchQuery.java | 8 +++--- 4 files changed, 32 insertions(+), 7 deletions(-) diff --git a/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java b/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java index 5bd2785ba04..3a4a4101ca6 100644 --- a/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java +++ b/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java @@ -32,6 +32,8 @@ public abstract class AbstractRedisService implements RedisService { private static final String ATLAS_METASTORE_SERVICE = "atlas-metastore-service"; RedissonClient redisClient; + + RedissonClient searchContextCacheRedisClient; Map keyLockMap; Configuration atlasConfig; long waitTimeInMS; @@ -74,20 +76,20 @@ public void releaseDistributedLock(String key) { @Override public String getValue(String key) { // If value doesn't exist, return null else return the value - return (String) redisClient.getBucket(convertToNamespace(key)).get(); + return (String) searchContextCacheRedisClient.getBucket(convertToNamespace(key)).get(); } @Override public String putValue(String key, String value) { // Put the value in the redis cache with TTL - redisClient.getBucket(convertToNamespace(key)).set(value, 30, TimeUnit.SECONDS); + searchContextCacheRedisClient.getBucket(convertToNamespace(key)).set(value, 30, TimeUnit.SECONDS); return value; } @Override public void removeValue(String key) { // Remove the value from the redis cache - redisClient.getBucket(convertToNamespace(key)).delete(); + searchContextCacheRedisClient.getBucket(convertToNamespace(key)).delete(); } private String getHostAddress() throws UnknownHostException { @@ -136,6 +138,26 @@ protected Config getProdConfig() throws AtlasException { return config; } + protected Config getSearchContextCacheConfig() throws AtlasException { + Config config = new Config(); + config.useSentinelServers() + .setClientName(ATLAS_METASTORE_SERVICE) + .setReadMode(ReadMode.MASTER_SLAVE) + .setCheckSentinelsList(false) + .setKeepAlive(true) + .setMasterConnectionMinimumIdleSize(10) + .setMasterConnectionPoolSize(20) + .setSlaveConnectionMinimumIdleSize(10) + .setSlaveConnectionPoolSize(20) + .setMasterName(atlasConfig.getString(ATLAS_REDIS_MASTER_NAME)) + .addSentinelAddress(formatUrls(atlasConfig.getStringArray(ATLAS_REDIS_SENTINEL_URLS))) + .setUsername(atlasConfig.getString(ATLAS_REDIS_USERNAME)) + .setPassword(atlasConfig.getString(ATLAS_REDIS_PASSWORD)) + .setTimeout(100) //Setting UP timeout to 100ms + .setRetryAttempts(0); + return config; + } + private String[] formatUrls(String[] urls) throws IllegalArgumentException { if (ArrayUtils.isEmpty(urls)) { getLogger().error("Invalid redis cluster urls"); diff --git a/common/src/main/java/org/apache/atlas/service/redis/RedisServiceImpl.java b/common/src/main/java/org/apache/atlas/service/redis/RedisServiceImpl.java index 42dec6fa783..c553cacfdbd 100644 --- a/common/src/main/java/org/apache/atlas/service/redis/RedisServiceImpl.java +++ b/common/src/main/java/org/apache/atlas/service/redis/RedisServiceImpl.java @@ -3,6 +3,7 @@ import org.apache.atlas.AtlasException; import org.apache.atlas.annotation.ConditionalOnAtlasProperty; import org.redisson.Redisson; +import org.redisson.config.Config; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -18,6 +19,7 @@ public class RedisServiceImpl extends AbstractRedisService{ @PostConstruct public void init() throws AtlasException { redisClient = Redisson.create(getProdConfig()); + searchContextCacheRedisClient = Redisson.create(getSearchContextCacheConfig()); LOG.info("Sentinel redis client created successfully."); } diff --git a/common/src/main/java/org/apache/atlas/service/redis/RedisServiceLocalImpl.java b/common/src/main/java/org/apache/atlas/service/redis/RedisServiceLocalImpl.java index 1ddad0f3e80..61db05ee30e 100644 --- a/common/src/main/java/org/apache/atlas/service/redis/RedisServiceLocalImpl.java +++ b/common/src/main/java/org/apache/atlas/service/redis/RedisServiceLocalImpl.java @@ -18,6 +18,7 @@ public class RedisServiceLocalImpl extends AbstractRedisService { @PostConstruct public void init() throws AtlasException { redisClient = Redisson.create(getLocalConfig()); + searchContextCacheRedisClient = Redisson.create(getLocalConfig()); LOG.info("Local redis client created successfully."); } diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java index 51e18379751..02686c39a7a 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java @@ -180,13 +180,13 @@ private DirectIndexQueryResult performAsyncDirectIndexQuery(SearchParams searchP if(contextIdExists) { // If the search context id and greater sequence no is present, then we need to delete the previous search context async try { - processRequestWithSameSearchContextId(searchParams); + CompletableFuture.runAsync(() -> processRequestWithSameSearchContextId(searchParams)); } catch (Exception e) { LOG.error("Failed to process the request with the same search context ID {}", e.getMessage()); } } AsyncQueryResult response = submitAsyncSearch(searchParams, false).get(); - + //Sleep for 5 seconds to allow ES to process the request if(response.isRunning()) { /* * If the response is still running, then we need to wait for the response @@ -199,7 +199,7 @@ private DirectIndexQueryResult performAsyncDirectIndexQuery(SearchParams searchP Integer searchContextSequenceNo = searchParams.getSearchContextSequenceNo(); if (contextIdExists) { try { - SearchContextCache.put(searchContextId, searchContextSequenceNo, esSearchId); + CompletableFuture.runAsync(() -> SearchContextCache.put(searchContextId, searchContextSequenceNo, esSearchId)); } catch (Exception e) { LOG.error("Failed to update the search context cache {}", e.getMessage()); } @@ -220,7 +220,7 @@ private DirectIndexQueryResult performAsyncDirectIndexQuery(SearchParams searchP if (contextIdExists) { // If the search context id is present, then we need to remove the search context from the cache try { - SearchContextCache.remove(searchParams.getSearchContextId()); + CompletableFuture.runAsync(() -> SearchContextCache.remove(searchParams.getSearchContextId())); } catch (Exception e) { LOG.error("Failed to remove the search context from the cache {}", e.getMessage()); } From ea76a4dae4aca833b459daafe53c788bfc5a547b Mon Sep 17 00:00:00 2001 From: Rajat Movaliya Date: Wed, 1 May 2024 16:13:42 +0530 Subject: [PATCH 092/316] update minor changes --- .../apache/atlas/model/instance/AtlasEntityHeader.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java index f5f060349dd..b6376a83425 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java @@ -74,7 +74,6 @@ public class AtlasEntityHeader extends AtlasStruct implements Serializable { private Integer depth = null; private Map collapse = null; - public AtlasEntityHeader() { this(null, null); } @@ -157,12 +156,12 @@ public void setGuid(String guid) { } public Integer getDepth() { - return depth; - } + return depth; + } public void setDepth(Integer depth) { - this.depth = depth; - } + this.depth = depth; + } public AtlasEntity.Status getStatus() { return status; From 2d8cbd97c9dddee38c7ec99abd7ecae40e3f5138 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Wed, 1 May 2024 16:18:31 +0530 Subject: [PATCH 093/316] fix: just use new config --- .../org/apache/atlas/service/redis/AbstractRedisService.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java b/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java index 3a4a4101ca6..bbb8f802429 100644 --- a/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java +++ b/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java @@ -121,7 +121,7 @@ Config getLocalConfig() throws AtlasException { } protected Config getProdConfig() throws AtlasException { - Config config = initAtlasConfig(); + Config config = new Config(); config.useSentinelServers() .setClientName(ATLAS_METASTORE_SERVICE) .setReadMode(ReadMode.MASTER_SLAVE) @@ -135,6 +135,7 @@ protected Config getProdConfig() throws AtlasException { .addSentinelAddress(formatUrls(atlasConfig.getStringArray(ATLAS_REDIS_SENTINEL_URLS))) .setUsername(atlasConfig.getString(ATLAS_REDIS_USERNAME)) .setPassword(atlasConfig.getString(ATLAS_REDIS_PASSWORD)); + return config; } From 1cb04d5f3ef05514f75f4abcf0c1be55924150b0 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Wed, 1 May 2024 16:30:32 +0530 Subject: [PATCH 094/316] fix: update config --- .../apache/atlas/service/redis/AbstractRedisService.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java b/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java index bbb8f802429..967c8fb5ec4 100644 --- a/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java +++ b/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java @@ -120,8 +120,8 @@ Config getLocalConfig() throws AtlasException { return config; } - protected Config getProdConfig() throws AtlasException { - Config config = new Config(); + Config getProdConfig() throws AtlasException { + Config config = initAtlasConfig(); config.useSentinelServers() .setClientName(ATLAS_METASTORE_SERVICE) .setReadMode(ReadMode.MASTER_SLAVE) @@ -135,11 +135,10 @@ protected Config getProdConfig() throws AtlasException { .addSentinelAddress(formatUrls(atlasConfig.getStringArray(ATLAS_REDIS_SENTINEL_URLS))) .setUsername(atlasConfig.getString(ATLAS_REDIS_USERNAME)) .setPassword(atlasConfig.getString(ATLAS_REDIS_PASSWORD)); - return config; } - protected Config getSearchContextCacheConfig() throws AtlasException { + Config getSearchContextCacheConfig() throws AtlasException { Config config = new Config(); config.useSentinelServers() .setClientName(ATLAS_METASTORE_SERVICE) From d38091a62a40f00d5820e539266987d9930c0cfb Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Wed, 1 May 2024 16:52:37 +0530 Subject: [PATCH 095/316] feat: update conetxt naming in redis client --- .../org/apache/atlas/service/redis/AbstractRedisService.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java b/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java index 967c8fb5ec4..59703f5f520 100644 --- a/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java +++ b/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java @@ -32,7 +32,6 @@ public abstract class AbstractRedisService implements RedisService { private static final String ATLAS_METASTORE_SERVICE = "atlas-metastore-service"; RedissonClient redisClient; - RedissonClient searchContextCacheRedisClient; Map keyLockMap; Configuration atlasConfig; @@ -138,10 +137,10 @@ Config getProdConfig() throws AtlasException { return config; } - Config getSearchContextCacheConfig() throws AtlasException { + Config getSearchContextCacheConfig() { Config config = new Config(); config.useSentinelServers() - .setClientName(ATLAS_METASTORE_SERVICE) + .setClientName(ATLAS_METASTORE_SERVICE+"-searchContextCache") .setReadMode(ReadMode.MASTER_SLAVE) .setCheckSentinelsList(false) .setKeepAlive(true) From df3864595ab9958245570d3458cbb3c6998f8b66 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Thu, 2 May 2024 18:32:19 +0530 Subject: [PATCH 096/316] fix: set redis timeout as 10 sec --- .../org/apache/atlas/service/redis/AbstractRedisService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java b/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java index 59703f5f520..8238d32fbb6 100644 --- a/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java +++ b/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java @@ -152,7 +152,7 @@ Config getSearchContextCacheConfig() { .addSentinelAddress(formatUrls(atlasConfig.getStringArray(ATLAS_REDIS_SENTINEL_URLS))) .setUsername(atlasConfig.getString(ATLAS_REDIS_USERNAME)) .setPassword(atlasConfig.getString(ATLAS_REDIS_PASSWORD)) - .setTimeout(100) //Setting UP timeout to 100ms + .setTimeout(10) //Setting UP timeout to 10ms .setRetryAttempts(0); return config; } From 0f68575311822f621143771dddc68cd5de342866 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Thu, 2 May 2024 18:34:16 +0530 Subject: [PATCH 097/316] Revert "Revert "feat: Search cancellation from Metastore with global context cache using Redis"" --- .../service/redis/AbstractRedisService.java | 26 +++++- .../service/redis/NoRedisServiceImpl.java | 15 ++++ .../atlas/service/redis/RedisService.java | 6 ++ .../service/redis/RedisServiceLocalImpl.java | 15 ++++ graphdb/janus/pom.xml | 4 + .../janus/AtlasElasticsearchQuery.java | 56 +++++++++---- .../graphdb/janus/SearchContextCache.java | 81 +++++++++++++------ 7 files changed, 161 insertions(+), 42 deletions(-) diff --git a/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java b/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java index 0ae20a60cc4..5bd2785ba04 100644 --- a/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java +++ b/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java @@ -71,6 +71,25 @@ public void releaseDistributedLock(String key) { } } + @Override + public String getValue(String key) { + // If value doesn't exist, return null else return the value + return (String) redisClient.getBucket(convertToNamespace(key)).get(); + } + + @Override + public String putValue(String key, String value) { + // Put the value in the redis cache with TTL + redisClient.getBucket(convertToNamespace(key)).set(value, 30, TimeUnit.SECONDS); + return value; + } + + @Override + public void removeValue(String key) { + // Remove the value from the redis cache + redisClient.getBucket(convertToNamespace(key)).delete(); + } + private String getHostAddress() throws UnknownHostException { return InetAddress.getLocalHost().getHostAddress(); } @@ -85,6 +104,11 @@ private Config initAtlasConfig() throws AtlasException { return redisConfig; } + private String convertToNamespace(String key){ + // Append key with namespace :atlas + return "atlas:"+key; + } + Config getLocalConfig() throws AtlasException { Config config = initAtlasConfig(); config.useSingleServer() @@ -94,7 +118,7 @@ Config getLocalConfig() throws AtlasException { return config; } - Config getProdConfig() throws AtlasException { + protected Config getProdConfig() throws AtlasException { Config config = initAtlasConfig(); config.useSentinelServers() .setClientName(ATLAS_METASTORE_SERVICE) diff --git a/common/src/main/java/org/apache/atlas/service/redis/NoRedisServiceImpl.java b/common/src/main/java/org/apache/atlas/service/redis/NoRedisServiceImpl.java index 96a8fadc99f..5bc089d9557 100644 --- a/common/src/main/java/org/apache/atlas/service/redis/NoRedisServiceImpl.java +++ b/common/src/main/java/org/apache/atlas/service/redis/NoRedisServiceImpl.java @@ -29,6 +29,21 @@ public void releaseDistributedLock(String key) { //do nothing } + @Override + public String getValue(String key) { + return null; + } + + @Override + public String putValue(String key, String value) { + return null; + } + + @Override + public void removeValue(String key) { + + } + @Override public Logger getLogger() { return LOG; diff --git a/common/src/main/java/org/apache/atlas/service/redis/RedisService.java b/common/src/main/java/org/apache/atlas/service/redis/RedisService.java index 1475f93e832..6cb2f04ada3 100644 --- a/common/src/main/java/org/apache/atlas/service/redis/RedisService.java +++ b/common/src/main/java/org/apache/atlas/service/redis/RedisService.java @@ -8,6 +8,12 @@ public interface RedisService { void releaseDistributedLock(String key); + String getValue(String key); + + String putValue(String key, String value); + + void removeValue(String key); + Logger getLogger(); } diff --git a/common/src/main/java/org/apache/atlas/service/redis/RedisServiceLocalImpl.java b/common/src/main/java/org/apache/atlas/service/redis/RedisServiceLocalImpl.java index 2eb774920ef..1ddad0f3e80 100644 --- a/common/src/main/java/org/apache/atlas/service/redis/RedisServiceLocalImpl.java +++ b/common/src/main/java/org/apache/atlas/service/redis/RedisServiceLocalImpl.java @@ -21,6 +21,21 @@ public void init() throws AtlasException { LOG.info("Local redis client created successfully."); } + @Override + public String getValue(String key) { + return null; + } + + @Override + public String putValue(String key, String value) { + return null; + } + + @Override + public void removeValue(String key) { + + } + @Override public Logger getLogger() { return LOG; diff --git a/graphdb/janus/pom.xml b/graphdb/janus/pom.xml index 5daef76b4ee..75c9079eee1 100644 --- a/graphdb/janus/pom.xml +++ b/graphdb/janus/pom.xml @@ -282,6 +282,10 @@ mockito-all test + + org.apache.atlas + atlas-server-api + diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java index 9d2e2489e8a..51e18379751 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java @@ -19,12 +19,14 @@ import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.RequestContext; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.discovery.SearchParams; import org.apache.atlas.repository.graphdb.AtlasIndexQuery; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.graphdb.DirectIndexQueryResult; import org.apache.atlas.type.AtlasType; +import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.NotImplementedException; import org.apache.commons.lang.StringUtils; @@ -171,11 +173,17 @@ private Map runQueryWithLowLevelClient(String query) throws Atla } private DirectIndexQueryResult performAsyncDirectIndexQuery(SearchParams searchParams) throws AtlasBaseException, IOException { + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("performAsyncDirectIndexQuery"); DirectIndexQueryResult result = null; + boolean contextIdExists = StringUtils.isNotEmpty(searchParams.getSearchContextId()) && searchParams.getSearchContextSequenceNo() != null; try { - if(StringUtils.isNotEmpty(searchParams.getSearchContextId()) && searchParams.getSearchContextSequenceNo() != null) { + if(contextIdExists) { // If the search context id and greater sequence no is present, then we need to delete the previous search context async - processRequestWithSameSearchContextId(searchParams); + try { + processRequestWithSameSearchContextId(searchParams); + } catch (Exception e) { + LOG.error("Failed to process the request with the same search context ID {}", e.getMessage()); + } } AsyncQueryResult response = submitAsyncSearch(searchParams, false).get(); @@ -189,8 +197,12 @@ private DirectIndexQueryResult performAsyncDirectIndexQuery(SearchParams searchP String esSearchId = response.getId(); String searchContextId = searchParams.getSearchContextId(); Integer searchContextSequenceNo = searchParams.getSearchContextSequenceNo(); - if (StringUtils.isNotEmpty(searchContextId) && searchContextSequenceNo != null) { - SearchContextCache.put(searchContextId, searchContextSequenceNo, esSearchId); + if (contextIdExists) { + try { + SearchContextCache.put(searchContextId, searchContextSequenceNo, esSearchId); + } catch (Exception e) { + LOG.error("Failed to update the search context cache {}", e.getMessage()); + } } response = getAsyncSearchResponse(searchParams, esSearchId).get(); if (response == null) { @@ -204,6 +216,16 @@ private DirectIndexQueryResult performAsyncDirectIndexQuery(SearchParams searchP }catch (Exception e) { LOG.error("Failed to execute direct query on ES {}", e.getMessage()); throw new AtlasBaseException(AtlasErrorCode.INDEX_SEARCH_FAILED, e.getMessage()); + } finally { + if (contextIdExists) { + // If the search context id is present, then we need to remove the search context from the cache + try { + SearchContextCache.remove(searchParams.getSearchContextId()); + } catch (Exception e) { + LOG.error("Failed to remove the search context from the cache {}", e.getMessage()); + } + } + RequestContext.get().endMetricRecord(metric); } return result; } @@ -218,17 +240,21 @@ private DirectIndexQueryResult performAsyncDirectIndexQuery(SearchParams searchP * We also need to check if the search ID exists and delete if necessary */ private void processRequestWithSameSearchContextId(SearchParams searchParams) { - // Extract search context ID and sequence number - String currentSearchContextId = searchParams.getSearchContextId(); - Integer currentSequenceNumber = searchParams.getSearchContextSequenceNo(); - // Get the search ID from the cache if sequence number is greater than the current sequence number - String previousESSearchId = SearchContextCache.getESAsyncSearchIdFromContextCache(currentSearchContextId, currentSequenceNumber); - - if (StringUtils.isNotEmpty(previousESSearchId)) { - LOG.debug("Deleting the previous async search response with ID {}", previousESSearchId); - // If the search ID exists, then we need to delete the search context - deleteAsyncSearchResponse(previousESSearchId); - SearchContextCache.remove(currentSearchContextId); + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("processRequestWithSameSearchContextId"); + try { + // Extract search context ID and sequence number + String currentSearchContextId = searchParams.getSearchContextId(); + Integer currentSequenceNumber = searchParams.getSearchContextSequenceNo(); + // Get the search ID from the cache if sequence number is greater than the current sequence number + String previousESSearchId = SearchContextCache.getESAsyncSearchIdFromContextCache(currentSearchContextId, currentSequenceNumber); + + if (StringUtils.isNotEmpty(previousESSearchId)) { + LOG.debug("Deleting the previous async search response with ID {}", previousESSearchId); + // If the search ID exists, then we need to delete the search context + deleteAsyncSearchResponse(previousESSearchId); + } + } finally { + RequestContext.get().endMetricRecord(metric); } } diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/SearchContextCache.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/SearchContextCache.java index 1780e5d1f90..fc627ef51f6 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/SearchContextCache.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/SearchContextCache.java @@ -2,48 +2,77 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; +import org.apache.atlas.RequestContext; +import org.apache.atlas.service.redis.AbstractRedisService; +import org.apache.atlas.service.redis.RedisService; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.slf4j.Logger; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Component; import java.util.HashMap; import java.util.Map; import java.util.concurrent.TimeUnit; +@Component public class SearchContextCache { - private static final Cache> searchContextCache = CacheBuilder.newBuilder() - .maximumSize(200) - .expireAfterWrite(30, TimeUnit.SECONDS) - .build(); + private static RedisService redisService = null; + + public static final String INVALID_SEQUENCE = "invalid_sequence"; + + + public SearchContextCache(@Qualifier("redisServiceImpl") RedisService redisService) { + SearchContextCache.redisService = redisService; + } + public static void put(String key, Integer sequence, String esAsyncId) { - HashMap entry = new HashMap<>(); - entry.put(sequence, esAsyncId); - searchContextCache.put(key, entry); + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("putInCache"); + try { + // Build the string in format `sequence/esAsyncId` and store it in redis + String val = sequence + "/" + esAsyncId; + redisService.putValue(key, val); + } finally { + RequestContext.get().endMetricRecord(metric); + } } - public static HashMap get(String key){ - return searchContextCache.getIfPresent(key); + public static String get(String key){ + return redisService.getValue(key); } public static String getESAsyncSearchIdFromContextCache(String key, Integer sequence){ - //Get the context cache for the given key - HashMap contextCache = get(key); - if(contextCache == null || sequence == null){ - return null; - } - //Find the highest sequence number - int maxStoredSequence = 0; - for (Integer seq : contextCache.keySet()) { - if (seq > maxStoredSequence) { - maxStoredSequence = seq; + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("getESAsyncSearchIdFromContextCache"); + try { + //Get the context cache for the given key + String contextCache = get(key); + if(contextCache == null || sequence == null){ + return null; + } + // Split the context cache to get the sequence and ESAsyncId + String[] contextCacheSplit = contextCache.split("/"); + if(contextCacheSplit.length != 2){ + return null; } + int seq = Integer.parseInt(contextCacheSplit[0]); + if(sequence > seq){ + return contextCacheSplit[1]; + } else if (sequence < seq) { + return INVALID_SEQUENCE; + } + return null; + } finally { + RequestContext.get().endMetricRecord(metric); } - //If the given sequence is greater than the max stored sequence, return the ESAsyncId else return null - return sequence > maxStoredSequence ? contextCache.getOrDefault(maxStoredSequence, null) : null; - } - public static void remove(String key) { - searchContextCache.invalidate(key); } + public static void remove(String key) { + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("removeFromCache"); + try { + redisService.removeValue(key); + } finally { + RequestContext.get().endMetricRecord(metric); + } - public static void clear() { - searchContextCache.cleanUp(); } } + From 1cfb2ede80324f359c84abba95c5c01e7523de2b Mon Sep 17 00:00:00 2001 From: Rajat Movaliya Date: Thu, 2 May 2024 18:34:16 +0530 Subject: [PATCH 098/316] add traversalOrder attribute --- .../model/instance/AtlasEntityHeader.java | 21 +++++++-------- .../atlas/discovery/EntityLineageService.java | 26 ++++++++++++------- 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java index f5f060349dd..7cb3f21b1c3 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java @@ -72,6 +72,7 @@ public class AtlasEntityHeader extends AtlasStruct implements Serializable { private Date updateTime = null; private String deleteHandler = null; private Integer depth = null; + private Integer traversalOrder = null; private Map collapse = null; @@ -148,21 +149,17 @@ public AtlasEntityHeader(AtlasEntity entity) { } } - public String getGuid() { - return guid; - } + public String getGuid() { return guid; } - public void setGuid(String guid) { - this.guid = guid; - } + public void setGuid(String guid) { this.guid = guid; } - public Integer getDepth() { - return depth; - } + public Integer getDepth() { return depth; } - public void setDepth(Integer depth) { - this.depth = depth; - } + public void setDepth(Integer depth) { this.depth = depth; } + + public Integer getTraversalOrder() { return traversalOrder; } + + public void setTraversalOrder(Integer traversalOrder) { this.traversalOrder = traversalOrder; } public AtlasEntity.Status getStatus() { return status; diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index 3ec77439061..353232cdbff 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -340,7 +340,7 @@ private void traverseEdgesOnDemand(Iterator processEdges, boolean isI if (incrementAndCheckIfRelationsLimitReached(processEdge, isInputEdge, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, direction)) { break; } else { - addEdgeToResult(processEdge, ret, atlasLineageOnDemandContext, nextLevel); + addEdgeToResult(processEdge, ret, atlasLineageOnDemandContext, nextLevel, entitiesTraversed); } String inGuid = AtlasGraphUtilsV2.getIdFromVertex(datasetVertex); @@ -389,7 +389,7 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i else continue; } else { - addEdgeToResult(incomingEdge, ret, atlasLineageOnDemandContext, level); + addEdgeToResult(incomingEdge, ret, atlasLineageOnDemandContext, level, entitiesTraversed); } AtlasPerfMetrics.MetricRecorder traverseEdgesOnDemandGetEdgesOut = RequestContext.get().startMetricRecord("traverseEdgesOnDemandGetEdgesOut"); @@ -417,7 +417,7 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i else continue; } else { - addEdgeToResult(outgoingEdge, ret, atlasLineageOnDemandContext, nextLevel); + addEdgeToResult(outgoingEdge, ret, atlasLineageOnDemandContext, nextLevel, entitiesTraversed); entitiesTraversed.incrementAndGet(); if (isEntityTraversalLimitReached(entitiesTraversed)) setEntityLimitReachedFlag(isInput, ret); @@ -878,9 +878,9 @@ private void addEdgeToResult(AtlasEdge edge, AtlasLineageInfo lineageInfo, } } - private void addEdgeToResult(AtlasEdge edge, AtlasLineageOnDemandInfo lineageInfo, AtlasLineageOnDemandContext atlasLineageOnDemandContext, int level) throws AtlasBaseException { + private void addEdgeToResult(AtlasEdge edge, AtlasLineageOnDemandInfo lineageInfo, AtlasLineageOnDemandContext atlasLineageOnDemandContext, int level, AtomicInteger entitiesTraversed) throws AtlasBaseException { if (!lineageContainsVisitedEdgeV2(lineageInfo, edge)) { - processEdge(edge, lineageInfo, atlasLineageOnDemandContext, level); + processEdge(edge, lineageInfo, atlasLineageOnDemandContext, level, entitiesTraversed); } } @@ -1450,11 +1450,11 @@ private void processEdge(final AtlasEdge edge, final Map entities, final Set relations, final Set visitedEdges, final Set attributes, int level) throws AtlasBaseException { + private void processEdge(final AtlasEdge edge, final Map entities, final Set relations, final Set visitedEdges, final Set attributes, int level, AtomicInteger entitiesTraversed) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processEdge"); AtlasVertex inVertex = edge.getInVertex(); AtlasVertex outVertex = edge.getOutVertex(); @@ -1466,12 +1466,18 @@ private void processEdge(final AtlasEdge edge, final Map Date: Thu, 2 May 2024 18:54:14 +0530 Subject: [PATCH 099/316] removed QN nanoId changes --- .../store/graph/v2/AtlasEntityStoreV2.java | 4 ++-- .../datamesh/DataProductPreProcessor.java | 17 +---------------- .../datamesh/DomainPreProcessor.java | 18 +----------------- 3 files changed, 4 insertions(+), 35 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 472c29cc468..0865eb47074 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -1803,11 +1803,11 @@ public PreProcessor getPreProcessor(String typeName) { break; case DATA_DOMAIN_ENTITY_TYPE: - preProcessor = new DomainPreProcessor(typeRegistry, entityRetriever, graph, entityGraphMapper); + preProcessor = new DomainPreProcessor(typeRegistry, entityRetriever, graph); break; case DATA_PRODUCT_ENTITY_TYPE: - preProcessor = new DataProductPreProcessor(typeRegistry, entityRetriever, graph, entityGraphMapper); + preProcessor = new DataProductPreProcessor(typeRegistry, entityRetriever, graph); break; case QUERY_ENTITY_TYPE: diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 0eca39f9ec5..f2daab6d9be 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -6,7 +6,6 @@ import org.apache.atlas.model.instance.*; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; -import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; import org.apache.atlas.type.AtlasTypeRegistry; @@ -28,7 +27,7 @@ public class DataProductPreProcessor extends AbstractDomainPreProcessor { private AtlasEntityHeader parentDomain; private EntityMutationContext context; public DataProductPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, - AtlasGraph graph, EntityGraphMapper entityGraphMapper) { + AtlasGraph graph) { super(typeRegistry, entityRetriever, graph); } @@ -61,25 +60,11 @@ private void processCreateProduct(AtlasEntity entity, AtlasVertex vertex) throws AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreateProduct"); String productName = (String) entity.getAttribute(NAME); String parentDomainQualifiedName = (String) entity.getAttribute(PARENT_DOMAIN_QN); - Map customAttributes = new HashMap<>(); - customAttributes.put(MIGRATION_CUSTOM_ATTRIBUTE, "true"); productExists(productName, parentDomainQualifiedName); - String newQualifiedName = createQualifiedName(parentDomainQualifiedName); - - entity.setAttribute(QUALIFIED_NAME, newQualifiedName); - entity.setCustomAttributes(customAttributes); - RequestContext.get().endMetricRecord(metricRecorder); } - private static String createQualifiedName(String parentDomainQualifiedName) throws AtlasBaseException { - if (StringUtils.isEmpty(parentDomainQualifiedName)) { - throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Parent Domain Qualified Name cannot be empty or null"); - } - return parentDomainQualifiedName + "/product/" + getUUID(); - } - private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdateDomain"); String productName = (String) entity.getAttribute(NAME); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index 9d2620de09f..fd1fbe3550a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -30,7 +30,6 @@ import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; -import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; import org.apache.atlas.type.AtlasTypeRegistry; @@ -51,13 +50,11 @@ public class DomainPreProcessor extends AbstractDomainPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(DomainPreProcessor.class); private AtlasEntityHeader parentDomain; - private EntityGraphMapper entityGraphMapper; private EntityMutationContext context; public DomainPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, - AtlasGraph graph, EntityGraphMapper entityGraphMapper) { + AtlasGraph graph) { super(typeRegistry, entityRetriever, graph); - this.entityGraphMapper = entityGraphMapper; } @Override @@ -90,24 +87,11 @@ private void processCreateDomain(AtlasEntity entity, AtlasVertex vertex) throws AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreateDomain"); String domainName = (String) entity.getAttribute(NAME); String parentDomainQualifiedName = (String) entity.getAttribute(PARENT_DOMAIN_QN); - Map customAttributes = new HashMap<>(); - customAttributes.put(MIGRATION_CUSTOM_ATTRIBUTE, "true"); domainExists(domainName, parentDomainQualifiedName); - entity.setAttribute(QUALIFIED_NAME, createQualifiedName(parentDomainQualifiedName)); - entity.setCustomAttributes(customAttributes); - RequestContext.get().endMetricRecord(metricRecorder); } - private static String createQualifiedName(String parentDomainQualifiedName) { - if (StringUtils.isNotEmpty(parentDomainQualifiedName)) { - return parentDomainQualifiedName + "/domain/" + getUUID(); - } else{ - return "default/domain" + "/" + getUUID(); - } - } - private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdateDomain"); String domainName = (String) entity.getAttribute(NAME); From 93c08346bc1fbb1083fc679a90ccb026f79bc0bd Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Thu, 2 May 2024 21:04:34 +0530 Subject: [PATCH 100/316] feat: add failure metrics for redis --- .../service/metrics/MetricsRegistry.java | 2 +- .../metrics/MetricsRegistryServiceImpl.java | 67 ++++++++++--------- .../apache/atlas/utils/AtlasMetricType.java | 9 +++ .../apache/atlas/utils/AtlasPerfMetrics.java | 14 ++++ .../janus/AtlasElasticsearchQuery.java | 20 ++++-- .../java/org/apache/atlas/RequestContext.java | 2 +- 6 files changed, 77 insertions(+), 37 deletions(-) create mode 100644 common/src/main/java/org/apache/atlas/utils/AtlasMetricType.java diff --git a/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistry.java b/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistry.java index 9fdf5b903e5..ef8a49951cb 100644 --- a/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistry.java +++ b/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistry.java @@ -10,7 +10,7 @@ public interface MetricsRegistry { void collect(String requestId, String requestUri, AtlasPerfMetrics metrics); - void collectIndexsearch(String requestId, String requestUri, List applicationMetrics); + void collectIApplicationMetrics(String requestId, String requestUri, List applicationMetrics); void scrape(PrintWriter writer) throws IOException; diff --git a/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistryServiceImpl.java b/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistryServiceImpl.java index 1ae6d2980c4..e732ad13f0d 100644 --- a/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistryServiceImpl.java +++ b/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistryServiceImpl.java @@ -1,12 +1,10 @@ package org.apache.atlas.service.metrics; -import io.micrometer.core.instrument.Metrics; -import io.micrometer.core.instrument.Tag; -import io.micrometer.core.instrument.Tags; -import io.micrometer.core.instrument.Timer; +import io.micrometer.core.instrument.*; import io.micrometer.prometheus.PrometheusMeterRegistry; import org.apache.atlas.ApplicationProperties; import org.apache.atlas.AtlasException; +import org.apache.atlas.utils.AtlasMetricType; import org.apache.atlas.utils.AtlasPerfMetrics; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -64,35 +62,42 @@ public void collect(String requestId, String requestUri, AtlasPerfMetrics metric } } //Use this if you want to publish Histograms - public void collectIndexsearch(String requestId, String requestUri, List applicationMetrics){ + public void collectIApplicationMetrics(String requestId, String requestUri, List applicationMetrics){ try { for(AtlasPerfMetrics.Metric metric : applicationMetrics){ - Timer.builder(APPLICATION_LEVEL_METRICS_SUMMARY) - .serviceLevelObjectives( - Duration.ofMillis(500), - Duration.ofMillis(750), - Duration.ofMillis(1000), - Duration.ofMillis(1200), - Duration.ofMillis(1500), - Duration.ofSeconds(2), - Duration.ofSeconds(3), - Duration.ofSeconds(4), - Duration.ofSeconds(5), - Duration.ofSeconds(7), - Duration.ofSeconds(10), - Duration.ofSeconds(15), - Duration.ofSeconds(20), - Duration.ofSeconds(25), - Duration.ofSeconds(30), - Duration.ofSeconds(40), - Duration.ofSeconds(60), - Duration.ofSeconds(90), - Duration.ofSeconds(120), - Duration.ofSeconds(180) - ) - .publishPercentiles(PERCENTILES) - .tags(convertToMicrometerTags(metric.getTags())) - .register(getMeterRegistry()).record(metric.getTotalTimeMSecs(), TimeUnit.MILLISECONDS); + if (metric.getMetricType() == AtlasMetricType.COUNTER) { + Counter.builder(metric.getName()) + .tags(convertToMicrometerTags(metric.getTags())) + .register(getMeterRegistry()) + .increment(metric.getInvocations()); + } else { + Timer.builder(APPLICATION_LEVEL_METRICS_SUMMARY) + .serviceLevelObjectives( + Duration.ofMillis(500), + Duration.ofMillis(750), + Duration.ofMillis(1000), + Duration.ofMillis(1200), + Duration.ofMillis(1500), + Duration.ofSeconds(2), + Duration.ofSeconds(3), + Duration.ofSeconds(4), + Duration.ofSeconds(5), + Duration.ofSeconds(7), + Duration.ofSeconds(10), + Duration.ofSeconds(15), + Duration.ofSeconds(20), + Duration.ofSeconds(25), + Duration.ofSeconds(30), + Duration.ofSeconds(40), + Duration.ofSeconds(60), + Duration.ofSeconds(90), + Duration.ofSeconds(120), + Duration.ofSeconds(180) + ) + .publishPercentiles(PERCENTILES) + .tags(convertToMicrometerTags(metric.getTags())) + .register(getMeterRegistry()).record(metric.getTotalTimeMSecs(), TimeUnit.MILLISECONDS); + } } } catch (Exception e) { LOG.error("Failed to collect metrics", e); diff --git a/common/src/main/java/org/apache/atlas/utils/AtlasMetricType.java b/common/src/main/java/org/apache/atlas/utils/AtlasMetricType.java new file mode 100644 index 00000000000..6752b7fbd4d --- /dev/null +++ b/common/src/main/java/org/apache/atlas/utils/AtlasMetricType.java @@ -0,0 +1,9 @@ +package org.apache.atlas.utils; + +public enum AtlasMetricType { + COUNTER, + GAUGE, + HISTOGRAM, + METER, + TIMER +} diff --git a/common/src/main/java/org/apache/atlas/utils/AtlasPerfMetrics.java b/common/src/main/java/org/apache/atlas/utils/AtlasPerfMetrics.java index beebcb6ab19..dd8a101ad5a 100644 --- a/common/src/main/java/org/apache/atlas/utils/AtlasPerfMetrics.java +++ b/common/src/main/java/org/apache/atlas/utils/AtlasPerfMetrics.java @@ -104,6 +104,8 @@ long getElapsedTime() { public static class Metric { private final String name; + + private AtlasMetricType metricType; private long invocations = 0; private long totalTimeMSecs = 0; HashMap tags = new HashMap<>(); @@ -112,6 +114,14 @@ public Metric(String name) { this.name = name; } + public void setMetricType(AtlasMetricType metricType) { + this.metricType = metricType; + } + + public AtlasMetricType getMetricType() { + return metricType; + } + public String getName() { return name; } @@ -135,5 +145,9 @@ public HashMap getTags() { return tags; } + public void incrementInvocations() { + invocations++; + } + } } diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java index 02686c39a7a..8f4719ee41a 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java @@ -26,6 +26,7 @@ import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.graphdb.DirectIndexQueryResult; import org.apache.atlas.type.AtlasType; +import org.apache.atlas.utils.AtlasMetricType; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.NotImplementedException; @@ -48,6 +49,7 @@ import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.janusgraph.util.encoding.LongEncoding; +import org.redisson.client.RedisException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -180,7 +182,7 @@ private DirectIndexQueryResult performAsyncDirectIndexQuery(SearchParams searchP if(contextIdExists) { // If the search context id and greater sequence no is present, then we need to delete the previous search context async try { - CompletableFuture.runAsync(() -> processRequestWithSameSearchContextId(searchParams)); + processRequestWithSameSearchContextId(searchParams); } catch (Exception e) { LOG.error("Failed to process the request with the same search context ID {}", e.getMessage()); } @@ -240,8 +242,9 @@ private DirectIndexQueryResult performAsyncDirectIndexQuery(SearchParams searchP * We also need to check if the search ID exists and delete if necessary */ private void processRequestWithSameSearchContextId(SearchParams searchParams) { - AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("processRequestWithSameSearchContextId"); try { + AtlasPerfMetrics.Metric metric = new AtlasPerfMetrics.Metric("process_async_request_count"); + metric.setMetricType(AtlasMetricType.COUNTER); // Extract search context ID and sequence number String currentSearchContextId = searchParams.getSearchContextId(); Integer currentSequenceNumber = searchParams.getSearchContextSequenceNo(); @@ -253,8 +256,17 @@ private void processRequestWithSameSearchContextId(SearchParams searchParams) { // If the search ID exists, then we need to delete the search context deleteAsyncSearchResponse(previousESSearchId); } - } finally { - RequestContext.get().endMetricRecord(metric); + metric.incrementInvocations(); + RequestContext.get().addApplicationMetrics(metric); + } catch (RedisException e) { + AtlasPerfMetrics.Metric metric = new AtlasPerfMetrics.Metric("async_request_redis_failure_counter"); + metric.setMetricType(AtlasMetricType.COUNTER); + metric.incrementInvocations(); + LOG.error("Failed to process the request with the same search context ID {}", e.getMessage()); + RequestContext.get().addApplicationMetrics(metric); + } + catch (Exception e) { + LOG.error("Failed to process the request with the same search context ID {}", e.getMessage()); } } diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java b/server-api/src/main/java/org/apache/atlas/RequestContext.java index d27be167598..9fe4097f5f3 100644 --- a/server-api/src/main/java/org/apache/atlas/RequestContext.java +++ b/server-api/src/main/java/org/apache/atlas/RequestContext.java @@ -177,7 +177,7 @@ public void clearCache() { } if (CollectionUtils.isNotEmpty(applicationMetrics)) { if (Objects.nonNull(this.metricsRegistry)){ - this.metricsRegistry.collectIndexsearch(traceId, this.requestUri, applicationMetrics); + this.metricsRegistry.collectIApplicationMetrics(traceId, this.requestUri, applicationMetrics); } applicationMetrics.clear(); } From fce4132f05fd994d1b784c4453f5bdd281da3dce Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Thu, 2 May 2024 21:13:14 +0530 Subject: [PATCH 101/316] feat: redis client rename --- .../apache/atlas/service/redis/AbstractRedisService.java | 8 ++++---- .../org/apache/atlas/service/redis/RedisServiceImpl.java | 3 +-- .../apache/atlas/service/redis/RedisServiceLocalImpl.java | 2 +- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java b/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java index 8238d32fbb6..f7114f4f6a9 100644 --- a/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java +++ b/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java @@ -32,7 +32,7 @@ public abstract class AbstractRedisService implements RedisService { private static final String ATLAS_METASTORE_SERVICE = "atlas-metastore-service"; RedissonClient redisClient; - RedissonClient searchContextCacheRedisClient; + RedissonClient redisCacheClient; Map keyLockMap; Configuration atlasConfig; long waitTimeInMS; @@ -75,20 +75,20 @@ public void releaseDistributedLock(String key) { @Override public String getValue(String key) { // If value doesn't exist, return null else return the value - return (String) searchContextCacheRedisClient.getBucket(convertToNamespace(key)).get(); + return (String) redisCacheClient.getBucket(convertToNamespace(key)).get(); } @Override public String putValue(String key, String value) { // Put the value in the redis cache with TTL - searchContextCacheRedisClient.getBucket(convertToNamespace(key)).set(value, 30, TimeUnit.SECONDS); + redisCacheClient.getBucket(convertToNamespace(key)).set(value, 30, TimeUnit.SECONDS); return value; } @Override public void removeValue(String key) { // Remove the value from the redis cache - searchContextCacheRedisClient.getBucket(convertToNamespace(key)).delete(); + redisCacheClient.getBucket(convertToNamespace(key)).delete(); } private String getHostAddress() throws UnknownHostException { diff --git a/common/src/main/java/org/apache/atlas/service/redis/RedisServiceImpl.java b/common/src/main/java/org/apache/atlas/service/redis/RedisServiceImpl.java index c553cacfdbd..1dcc6f56bae 100644 --- a/common/src/main/java/org/apache/atlas/service/redis/RedisServiceImpl.java +++ b/common/src/main/java/org/apache/atlas/service/redis/RedisServiceImpl.java @@ -3,7 +3,6 @@ import org.apache.atlas.AtlasException; import org.apache.atlas.annotation.ConditionalOnAtlasProperty; import org.redisson.Redisson; -import org.redisson.config.Config; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -19,7 +18,7 @@ public class RedisServiceImpl extends AbstractRedisService{ @PostConstruct public void init() throws AtlasException { redisClient = Redisson.create(getProdConfig()); - searchContextCacheRedisClient = Redisson.create(getSearchContextCacheConfig()); + redisCacheClient = Redisson.create(getSearchContextCacheConfig()); LOG.info("Sentinel redis client created successfully."); } diff --git a/common/src/main/java/org/apache/atlas/service/redis/RedisServiceLocalImpl.java b/common/src/main/java/org/apache/atlas/service/redis/RedisServiceLocalImpl.java index 61db05ee30e..c625f105f13 100644 --- a/common/src/main/java/org/apache/atlas/service/redis/RedisServiceLocalImpl.java +++ b/common/src/main/java/org/apache/atlas/service/redis/RedisServiceLocalImpl.java @@ -18,7 +18,7 @@ public class RedisServiceLocalImpl extends AbstractRedisService { @PostConstruct public void init() throws AtlasException { redisClient = Redisson.create(getLocalConfig()); - searchContextCacheRedisClient = Redisson.create(getLocalConfig()); + redisCacheClient = Redisson.create(getLocalConfig()); LOG.info("Local redis client created successfully."); } From 5232e56278aca0ce19512caf79a78488b3c2bc87 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Thu, 2 May 2024 21:31:56 +0530 Subject: [PATCH 102/316] nit: remove red. excp handling and rename cfg --- .../apache/atlas/service/redis/AbstractRedisService.java | 4 ++-- .../org/apache/atlas/service/redis/RedisServiceImpl.java | 2 +- .../graphdb/janus/AtlasElasticsearchQuery.java | 9 +++++---- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java b/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java index f7114f4f6a9..08d8b21c66c 100644 --- a/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java +++ b/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java @@ -137,10 +137,10 @@ Config getProdConfig() throws AtlasException { return config; } - Config getSearchContextCacheConfig() { + Config getCacheImplConfig() { Config config = new Config(); config.useSentinelServers() - .setClientName(ATLAS_METASTORE_SERVICE+"-searchContextCache") + .setClientName(ATLAS_METASTORE_SERVICE+"-redisCache") .setReadMode(ReadMode.MASTER_SLAVE) .setCheckSentinelsList(false) .setKeepAlive(true) diff --git a/common/src/main/java/org/apache/atlas/service/redis/RedisServiceImpl.java b/common/src/main/java/org/apache/atlas/service/redis/RedisServiceImpl.java index 1dcc6f56bae..48f199473e0 100644 --- a/common/src/main/java/org/apache/atlas/service/redis/RedisServiceImpl.java +++ b/common/src/main/java/org/apache/atlas/service/redis/RedisServiceImpl.java @@ -18,7 +18,7 @@ public class RedisServiceImpl extends AbstractRedisService{ @PostConstruct public void init() throws AtlasException { redisClient = Redisson.create(getProdConfig()); - redisCacheClient = Redisson.create(getSearchContextCacheConfig()); + redisCacheClient = Redisson.create(getCacheImplConfig()); LOG.info("Sentinel redis client created successfully."); } diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java index 8f4719ee41a..5419da45c66 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java @@ -181,11 +181,7 @@ private DirectIndexQueryResult performAsyncDirectIndexQuery(SearchParams searchP try { if(contextIdExists) { // If the search context id and greater sequence no is present, then we need to delete the previous search context async - try { processRequestWithSameSearchContextId(searchParams); - } catch (Exception e) { - LOG.error("Failed to process the request with the same search context ID {}", e.getMessage()); - } } AsyncQueryResult response = submitAsyncSearch(searchParams, false).get(); //Sleep for 5 seconds to allow ES to process the request @@ -242,6 +238,8 @@ private DirectIndexQueryResult performAsyncDirectIndexQuery(SearchParams searchP * We also need to check if the search ID exists and delete if necessary */ private void processRequestWithSameSearchContextId(SearchParams searchParams) { + AtlasPerfMetrics.MetricRecorder funcMetric = RequestContext.get().startMetricRecord("processRequestWithSameSearchContextId"); + try { AtlasPerfMetrics.Metric metric = new AtlasPerfMetrics.Metric("process_async_request_count"); metric.setMetricType(AtlasMetricType.COUNTER); @@ -268,6 +266,9 @@ private void processRequestWithSameSearchContextId(SearchParams searchParams) { catch (Exception e) { LOG.error("Failed to process the request with the same search context ID {}", e.getMessage()); } + finally { + RequestContext.get().endMetricRecord(funcMetric); + } } /* From da3191cb62bbcb59d036739c1daab5c65c5d9121 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Thu, 2 May 2024 23:45:44 +0530 Subject: [PATCH 103/316] fix: remove redundant search metric --- .../graphdb/janus/AtlasElasticsearchQuery.java | 13 ++++--------- pom.xml | 2 +- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java index 5419da45c66..a291ac40a06 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java @@ -239,10 +239,7 @@ private DirectIndexQueryResult performAsyncDirectIndexQuery(SearchParams searchP */ private void processRequestWithSameSearchContextId(SearchParams searchParams) { AtlasPerfMetrics.MetricRecorder funcMetric = RequestContext.get().startMetricRecord("processRequestWithSameSearchContextId"); - try { - AtlasPerfMetrics.Metric metric = new AtlasPerfMetrics.Metric("process_async_request_count"); - metric.setMetricType(AtlasMetricType.COUNTER); // Extract search context ID and sequence number String currentSearchContextId = searchParams.getSearchContextId(); Integer currentSequenceNumber = searchParams.getSearchContextSequenceNo(); @@ -254,14 +251,12 @@ private void processRequestWithSameSearchContextId(SearchParams searchParams) { // If the search ID exists, then we need to delete the search context deleteAsyncSearchResponse(previousESSearchId); } - metric.incrementInvocations(); - RequestContext.get().addApplicationMetrics(metric); } catch (RedisException e) { - AtlasPerfMetrics.Metric metric = new AtlasPerfMetrics.Metric("async_request_redis_failure_counter"); - metric.setMetricType(AtlasMetricType.COUNTER); - metric.incrementInvocations(); + AtlasPerfMetrics.Metric failureCounter = new AtlasPerfMetrics.Metric("async_request_redis_failure_counter"); + failureCounter.setMetricType(AtlasMetricType.COUNTER); + failureCounter.incrementInvocations(); LOG.error("Failed to process the request with the same search context ID {}", e.getMessage()); - RequestContext.get().addApplicationMetrics(metric); + RequestContext.get().addApplicationMetrics(failureCounter); } catch (Exception e) { LOG.error("Failed to process the request with the same search context ID {}", e.getMessage()); diff --git a/pom.xml b/pom.xml index 1cc9aa70dc9..3abc39041b7 100644 --- a/pom.xml +++ b/pom.xml @@ -717,7 +717,7 @@ 4.4.13 2.12.4 2.12.4 - 0.6.03 + 0.6.04 0.5.3 1 3.1.0 From 686a9e23a652963d28641e63b41cdc8581355749 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Fri, 3 May 2024 00:01:07 +0530 Subject: [PATCH 104/316] feat: add local search context cache as backup --- .../graphdb/janus/SearchContextCache.java | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/SearchContextCache.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/SearchContextCache.java index fc627ef51f6..0d6e172cf65 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/SearchContextCache.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/SearchContextCache.java @@ -3,21 +3,22 @@ import com.google.common.cache.Cache; import com.google.common.cache.CacheBuilder; import org.apache.atlas.RequestContext; -import org.apache.atlas.service.redis.AbstractRedisService; import org.apache.atlas.service.redis.RedisService; import org.apache.atlas.utils.AtlasPerfMetrics; -import org.slf4j.Logger; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; -import java.util.HashMap; -import java.util.Map; import java.util.concurrent.TimeUnit; @Component public class SearchContextCache { private static RedisService redisService = null; + private static final Cache searchContextLocalCache = CacheBuilder.newBuilder() + .maximumSize(200) + .expireAfterWrite(30, TimeUnit.SECONDS) + .build(); + public static final String INVALID_SEQUENCE = "invalid_sequence"; @@ -32,12 +33,23 @@ public static void put(String key, Integer sequence, String esAsyncId) { // Build the string in format `sequence/esAsyncId` and store it in redis String val = sequence + "/" + esAsyncId; redisService.putValue(key, val); + searchContextLocalCache.put(key, val); } finally { RequestContext.get().endMetricRecord(metric); } } - public static String get(String key){ - return redisService.getValue(key); + public static String get(String key) { + String ret = null; + try { + ret = searchContextLocalCache.getIfPresent(key); + if (ret == null) { + ret = redisService.getValue(key); + } + return ret; + } catch (Exception e) { + return null; + } + } public static String getESAsyncSearchIdFromContextCache(String key, Integer sequence){ @@ -68,6 +80,7 @@ public static String getESAsyncSearchIdFromContextCache(String key, Integer sequ public static void remove(String key) { AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("removeFromCache"); try { + searchContextLocalCache.invalidate(key); redisService.removeValue(key); } finally { RequestContext.get().endMetricRecord(metric); From 5bf86cf3a0f95325b0a75d015ebcccbb17d1e1dc Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Fri, 3 May 2024 00:02:18 +0530 Subject: [PATCH 105/316] Update pom.xml --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 3abc39041b7..1cc9aa70dc9 100644 --- a/pom.xml +++ b/pom.xml @@ -717,7 +717,7 @@ 4.4.13 2.12.4 2.12.4 - 0.6.04 + 0.6.03 0.5.3 1 3.1.0 From c87cef6f7c229c4ef5375b617abffa1bcdb66d8d Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Fri, 3 May 2024 02:16:10 +0530 Subject: [PATCH 106/316] fix: update timeout to 100ms --- .../org/apache/atlas/service/redis/AbstractRedisService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java b/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java index 08d8b21c66c..40634e6dc19 100644 --- a/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java +++ b/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java @@ -152,7 +152,7 @@ Config getCacheImplConfig() { .addSentinelAddress(formatUrls(atlasConfig.getStringArray(ATLAS_REDIS_SENTINEL_URLS))) .setUsername(atlasConfig.getString(ATLAS_REDIS_USERNAME)) .setPassword(atlasConfig.getString(ATLAS_REDIS_PASSWORD)) - .setTimeout(10) //Setting UP timeout to 10ms + .setTimeout(100) //Setting UP timeout to 10ms .setRetryAttempts(0); return config; } From 3a46932a415b6c8b7f3bcca9f3688c78c2f7c97f Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Fri, 3 May 2024 03:22:35 +0530 Subject: [PATCH 107/316] feat: update the timeout to 50 --- .../org/apache/atlas/service/redis/AbstractRedisService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java b/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java index 40634e6dc19..b259c32c16c 100644 --- a/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java +++ b/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java @@ -152,7 +152,7 @@ Config getCacheImplConfig() { .addSentinelAddress(formatUrls(atlasConfig.getStringArray(ATLAS_REDIS_SENTINEL_URLS))) .setUsername(atlasConfig.getString(ATLAS_REDIS_USERNAME)) .setPassword(atlasConfig.getString(ATLAS_REDIS_PASSWORD)) - .setTimeout(100) //Setting UP timeout to 10ms + .setTimeout(50) //Setting UP timeout to 10ms .setRetryAttempts(0); return config; } From eacf1c701d9a787636fe377505112ddd4357408e Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Fri, 3 May 2024 03:44:07 +0530 Subject: [PATCH 108/316] fix: remove local cache --- .../graphdb/janus/SearchContextCache.java | 20 +------------------ 1 file changed, 1 insertion(+), 19 deletions(-) diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/SearchContextCache.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/SearchContextCache.java index 0d6e172cf65..0b80ce03344 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/SearchContextCache.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/SearchContextCache.java @@ -1,24 +1,13 @@ package org.apache.atlas.repository.graphdb.janus; - -import com.google.common.cache.Cache; -import com.google.common.cache.CacheBuilder; import org.apache.atlas.RequestContext; import org.apache.atlas.service.redis.RedisService; import org.apache.atlas.utils.AtlasPerfMetrics; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; - -import java.util.concurrent.TimeUnit; - @Component public class SearchContextCache { private static RedisService redisService = null; - private static final Cache searchContextLocalCache = CacheBuilder.newBuilder() - .maximumSize(200) - .expireAfterWrite(30, TimeUnit.SECONDS) - .build(); - public static final String INVALID_SEQUENCE = "invalid_sequence"; @@ -33,19 +22,13 @@ public static void put(String key, Integer sequence, String esAsyncId) { // Build the string in format `sequence/esAsyncId` and store it in redis String val = sequence + "/" + esAsyncId; redisService.putValue(key, val); - searchContextLocalCache.put(key, val); } finally { RequestContext.get().endMetricRecord(metric); } } public static String get(String key) { - String ret = null; try { - ret = searchContextLocalCache.getIfPresent(key); - if (ret == null) { - ret = redisService.getValue(key); - } - return ret; + return redisService.getValue(key); } catch (Exception e) { return null; } @@ -80,7 +63,6 @@ public static String getESAsyncSearchIdFromContextCache(String key, Integer sequ public static void remove(String key) { AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("removeFromCache"); try { - searchContextLocalCache.invalidate(key); redisService.removeValue(key); } finally { RequestContext.get().endMetricRecord(metric); From 328c6ec5262e37800474258ff7a5689711f9f39d Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Fri, 3 May 2024 08:38:35 +0530 Subject: [PATCH 109/316] optimized policy code and handled edge cases --- .../v2/preprocessor/PreProcessorUtils.java | 2 + .../datamesh/AbstractDomainPreProcessor.java | 67 +++++++++---------- .../datamesh/DataProductPreProcessor.java | 22 +++++- .../datamesh/DomainPreProcessor.java | 25 +++++-- 4 files changed, 73 insertions(+), 43 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index 6b282d17272..d1f76296bd4 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -44,6 +44,8 @@ public class PreProcessorUtils { //DataMesh models constants public static final String DATA_PRODUCT_TYPE = "DataProduct"; public static final String PARENT_DOMAIN = "parentDomain"; + public static final String SUB_DOMAIN_REL_TYPE = "subDomains"; + public static final String DATA_PRODUCT_REL_TYPE = "dataProducts"; public static final String PARENT_DOMAIN_QN = "parentDomainQualifiedName"; public static final String SUPER_DOMAIN_QN = "superDomainQualifiedName"; public static final String DATA_DOMAIN = "dataDomain"; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 4a8423851af..a9efa5e98f6 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -70,42 +70,40 @@ public abstract class AbstractDomainPreProcessor implements PreProcessor { protected void isAuthorized(AtlasEntityHeader sourceDomain, AtlasEntityHeader targetDomain) throws AtlasBaseException { - // source -> CREATE + UPDATE + DELETE - AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, sourceDomain), - "create on source Domain: ", sourceDomain.getAttribute(NAME)); + if(sourceDomain != null){ + // source -> CREATE + UPDATE + DELETE + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, sourceDomain), + "create on source Domain: ", sourceDomain.getAttribute(NAME)); - AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, sourceDomain), - "update on source Domain: ", sourceDomain.getAttribute(NAME)); + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, sourceDomain), + "update on source Domain: ", sourceDomain.getAttribute(NAME)); - AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_DELETE, sourceDomain), - "delete on source Domain: ", sourceDomain.getAttribute(NAME)); + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_DELETE, sourceDomain), + "delete on source Domain: ", sourceDomain.getAttribute(NAME)); + } + if(targetDomain != null){ + // target -> CREATE + UPDATE + DELETE + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, targetDomain), + "create on target Domain: ", targetDomain.getAttribute(NAME)); - // target -> CREATE + UPDATE + DELETE - AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, targetDomain), - "create on target Domain: ", targetDomain.getAttribute(NAME)); + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, targetDomain), + "update on target Domain: ", targetDomain.getAttribute(NAME)); - AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, targetDomain), - "update on target Domain: ", targetDomain.getAttribute(NAME)); - - AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_DELETE, targetDomain), - "delete on target Domain: ", targetDomain.getAttribute(NAME)); + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_DELETE, targetDomain), + "delete on target Domain: ", targetDomain.getAttribute(NAME)); + } } - protected void updatePolicy(String currentQualifiedName, String updatedQualifiedName, EntityMutationContext context) throws AtlasBaseException { + protected void updatePolicy(List currentResources, Map updatedPolicyResources, EntityMutationContext context) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("updateDomainPolicy"); try { - LOG.info("Updating policy for entity {}", currentQualifiedName); - Map updatedpolicyResources = new HashMap<>(); - - String currentResource = "entity:"+ currentQualifiedName; - String updatedResource = "entity:"+ updatedQualifiedName; - - updatedpolicyResources.put(currentResource, updatedResource); + LOG.info("Updating policies for entities {}", currentResources); - List policies = getPolicy(currentResource); + List policies = getPolicy(currentResources); if (CollectionUtils.isNotEmpty(policies)) { AtlasEntityType entityType = typeRegistry.getEntityTypeByName(POLICY_ENTITY_TYPE); + for (AtlasEntityHeader policy : policies) { AtlasVertex policyVertex = entityRetriever.getEntityVertex(policy.getGuid()); @@ -118,17 +116,18 @@ protected void updatePolicy(String currentQualifiedName, String updatedQualified List policyResources = (List) policyEntity.getAttribute(ATTR_POLICY_RESOURCES); - // Check if currentResource exists in the list before removing it - if (policyResources.contains(currentResource)) { - policyResources.remove(currentResource); - policyResources.add(updatedResource); - LOG.info("Policy resources after updating QN: {}", policyResources); - } else { - LOG.info("CurrentResource {} not found in the policy resources. Skipping update.", currentResource); + List updatedPolicyResourcesList = new ArrayList<>(); + + for (String resource : policyResources) { + if (updatedPolicyResources.containsKey(resource)) { + updatedPolicyResourcesList.add(updatedPolicyResources.get(resource)); + } else { + updatedPolicyResourcesList.add(resource); + } } policyVertex.removeProperty(ATTR_POLICY_RESOURCES); - policyEntity.setAttribute(ATTR_POLICY_RESOURCES, policyResources); + policyEntity.setAttribute(ATTR_POLICY_RESOURCES, updatedPolicyResourcesList); context.addUpdated(policyEntity.getGuid(), policyEntity, entityType, policyVertex); } } @@ -139,13 +138,13 @@ protected void updatePolicy(String currentQualifiedName, String updatedQualified } - protected List getPolicy(String resource) throws AtlasBaseException { + protected List getPolicy(List resources) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("getPolicy"); try { List> mustClauseList = new ArrayList<>(); mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", POLICY_ENTITY_TYPE))); mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); - mustClauseList.add(mapOf("terms", mapOf("policyResources", Arrays.asList(resource)))); + mustClauseList.add(mapOf("terms", mapOf("policyResources", resources))); Map bool = new HashMap<>(); bool.put("must", mustClauseList); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index f2daab6d9be..5a366286806 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -26,6 +26,8 @@ public class DataProductPreProcessor extends AbstractDomainPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(DataProductPreProcessor.class); private AtlasEntityHeader parentDomain; private EntityMutationContext context; + private List currentResources = new ArrayList<>(); + private Map updatedPolicyResources = new HashMap<>(); public DataProductPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, AtlasGraph graph) { super(typeRegistry, entityRetriever, graph); @@ -92,11 +94,16 @@ private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws } } + if(entity.hasRelationshipAttribute(DATA_DOMAIN) && entity.getRelationshipAttribute(DATA_DOMAIN) == null){ + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "DataProduct can only be moved to another Domain."); + } + if (!currentParentDomainQualifiedName.equals(newParentDomainQualifiedName) && entity.hasRelationshipAttribute(DATA_DOMAIN)) { //Auth check isAuthorized(currentParentDomainHeader, parentDomain); processMoveDataProductToAnotherDomain(entity, currentParentDomainQualifiedName, newParentDomainQualifiedName, vertexQnName, superDomainQualifiedName); + updatePolicy(this.currentResources, this.updatedPolicyResources, this.context); entity.setAttribute(PARENT_DOMAIN_QN, newParentDomainQualifiedName); } else { @@ -120,18 +127,29 @@ private void processMoveDataProductToAnotherDomain(AtlasEntity product, try { String productName = (String) product.getAttribute(NAME); + if(StringUtils.isEmpty(targetDomainQualifiedName)){ + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "DataProduct can only be moved to another Domain."); + } + LOG.info("Moving dataProduct {} to Domain {}", productName, targetDomainQualifiedName); productExists(productName, targetDomainQualifiedName); + if(StringUtils.isEmpty(sourceDomainQualifiedName)){ + sourceDomainQualifiedName = "default"; + } + String updatedQualifiedName = currentDataProductQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); product.setAttribute(QUALIFIED_NAME, updatedQualifiedName); product.setAttribute(PARENT_DOMAIN_QN, targetDomainQualifiedName); product.setAttribute(SUPER_DOMAIN_QN, superDomainQualifiedName); - //Update policy - updatePolicy(currentDataProductQualifiedName, updatedQualifiedName, context); + //Store domainPolicies and resources to be updated + String currentResource = "entity:"+ currentDataProductQualifiedName; + String updatedResource = "entity:"+ updatedQualifiedName; + this.updatedPolicyResources.put(currentResource, updatedResource); + this.currentResources.add(currentDataProductQualifiedName); LOG.info("Moved dataProduct {} to Domain {}", productName, targetDomainQualifiedName); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index fd1fbe3550a..b37969ac2e2 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -51,6 +51,8 @@ public class DomainPreProcessor extends AbstractDomainPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(DomainPreProcessor.class); private AtlasEntityHeader parentDomain; private EntityMutationContext context; + private List currentResources = new ArrayList<>(); + private Map updatedPolicyResources = new HashMap<>(); public DomainPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, AtlasGraph graph) { @@ -106,6 +108,10 @@ private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws String newParentDomainQualifiedName = ""; String superDomainQualifiedName = ""; + if(entity.hasRelationshipAttribute(SUB_DOMAIN_REL_TYPE) || entity.hasRelationshipAttribute(DATA_PRODUCT_REL_TYPE)){ + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Cannot update Domain with subDomains or dataProducts"); + } + if(currentParentDomain != null){ currentParentDomainHeader = entityRetriever.toAtlasEntityHeader(currentParentDomain.getGuid()); currentParentDomainQualifiedName = (String) currentParentDomainHeader.getAttribute(QUALIFIED_NAME); @@ -125,9 +131,7 @@ private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws } //Auth check - if(parentDomain != null && currentParentDomainHeader != null){ - isAuthorized(currentParentDomainHeader, parentDomain); - } + isAuthorized(currentParentDomainHeader, parentDomain); processMoveSubDomainToAnotherDomain(entity, vertex, currentParentDomainQualifiedName, newParentDomainQualifiedName, vertexQnName, superDomainQualifiedName); @@ -178,6 +182,7 @@ private void processMoveSubDomainToAnotherDomain(AtlasEntity domain, } moveChildrenToAnotherDomain(domainVertex, superDomainQualifiedName, null, sourceDomainQualifiedName, targetDomainQualifiedName); + updatePolicy(this.currentResources, this.updatedPolicyResources, this.context); LOG.info("Moved subDomain {} to Domain {}", domainName, targetDomainQualifiedName); @@ -209,8 +214,11 @@ private void moveChildrenToAnotherDomain(AtlasVertex childDomainVertex, childDomainVertex.setProperty(SUPER_DOMAIN_QN, superDomainQualifiedName); childDomainVertex.setProperty(PARENT_DOMAIN_QN, parentDomainQualifiedName); - //update policy - updatePolicy(currentDomainQualifiedName, updatedDomainQualifiedName, context); + //Store domainPolicies and resources to be updated + String currentResource = "entity:"+ currentDomainQualifiedName; + String updatedResource = "entity:"+ updatedDomainQualifiedName; + this.updatedPolicyResources.put(currentResource, updatedResource); + this.currentResources.add(currentDomainQualifiedName); //update system properties GraphHelper.setModifiedByAsString(childDomainVertex, RequestContext.get().getUser()); @@ -261,8 +269,11 @@ private void moveChildDataProductToAnotherDomain(AtlasVertex productVertex, productVertex.setProperty(PARENT_DOMAIN_QN, parentDomainQualifiedName); productVertex.setProperty(SUPER_DOMAIN_QN, superDomainQualifiedName); - //update policy - updatePolicy(currentQualifiedName, updatedQualifiedName, context); + //Store domainPolicies and resources to be updated + String currentResource = "entity:"+ currentQualifiedName; + String updatedResource = "entity:"+ updatedQualifiedName; + this.updatedPolicyResources.put(currentResource, updatedResource); + this.currentResources.add(currentQualifiedName); //update system properties GraphHelper.setModifiedByAsString(productVertex, RequestContext.get().getUser()); From 81922b57ecf74abf4ab1a76fb826d145ca6a2cb7 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Fri, 3 May 2024 10:17:08 +0530 Subject: [PATCH 110/316] init error resolved --- .../v2/preprocessor/datamesh/DataProductPreProcessor.java | 6 ++++-- .../graph/v2/preprocessor/datamesh/DomainPreProcessor.java | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 5a366286806..285a3963b8b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -26,11 +26,13 @@ public class DataProductPreProcessor extends AbstractDomainPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(DataProductPreProcessor.class); private AtlasEntityHeader parentDomain; private EntityMutationContext context; - private List currentResources = new ArrayList<>(); - private Map updatedPolicyResources = new HashMap<>(); + private List currentResources; + private Map updatedPolicyResources; public DataProductPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, AtlasGraph graph) { super(typeRegistry, entityRetriever, graph); + this.currentResources = new ArrayList<>(); + this.updatedPolicyResources = new HashMap<>(); } @Override diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index b37969ac2e2..0e53f89b556 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -51,12 +51,14 @@ public class DomainPreProcessor extends AbstractDomainPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(DomainPreProcessor.class); private AtlasEntityHeader parentDomain; private EntityMutationContext context; - private List currentResources = new ArrayList<>(); - private Map updatedPolicyResources = new HashMap<>(); + private List currentResources ; + private Map updatedPolicyResources; public DomainPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, AtlasGraph graph) { super(typeRegistry, entityRetriever, graph); + this.currentResources = new ArrayList<>(); + this.updatedPolicyResources = new HashMap<>(); } @Override From 7a3828b0114d9d3626e0c0d0fb40d3fdd8b3d0a9 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Fri, 3 May 2024 10:25:55 +0530 Subject: [PATCH 111/316] policy list populate error resolved --- .../v2/preprocessor/datamesh/DataProductPreProcessor.java | 2 +- .../graph/v2/preprocessor/datamesh/DomainPreProcessor.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 285a3963b8b..4f5681c072e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -151,7 +151,7 @@ private void processMoveDataProductToAnotherDomain(AtlasEntity product, String currentResource = "entity:"+ currentDataProductQualifiedName; String updatedResource = "entity:"+ updatedQualifiedName; this.updatedPolicyResources.put(currentResource, updatedResource); - this.currentResources.add(currentDataProductQualifiedName); + this.currentResources.add(currentResource); LOG.info("Moved dataProduct {} to Domain {}", productName, targetDomainQualifiedName); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index 0e53f89b556..7ba2addc3fb 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -220,7 +220,7 @@ private void moveChildrenToAnotherDomain(AtlasVertex childDomainVertex, String currentResource = "entity:"+ currentDomainQualifiedName; String updatedResource = "entity:"+ updatedDomainQualifiedName; this.updatedPolicyResources.put(currentResource, updatedResource); - this.currentResources.add(currentDomainQualifiedName); + this.currentResources.add(currentResource); //update system properties GraphHelper.setModifiedByAsString(childDomainVertex, RequestContext.get().getUser()); @@ -275,7 +275,7 @@ private void moveChildDataProductToAnotherDomain(AtlasVertex productVertex, String currentResource = "entity:"+ currentQualifiedName; String updatedResource = "entity:"+ updatedQualifiedName; this.updatedPolicyResources.put(currentResource, updatedResource); - this.currentResources.add(currentQualifiedName); + this.currentResources.add(currentResource); //update system properties GraphHelper.setModifiedByAsString(productVertex, RequestContext.get().getUser()); From e129a391c5d09802814ee990178f25f8e2cdfee3 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Fri, 3 May 2024 11:37:43 +0530 Subject: [PATCH 112/316] fix: review fix --- .../apache/atlas/service/metrics/MetricsRegistry.java | 2 +- .../service/metrics/MetricsRegistryServiceImpl.java | 2 +- .../atlas/service/redis/AbstractRedisService.java | 2 +- .../graphdb/janus/AtlasElasticsearchQuery.java | 10 +++------- .../repository/graphdb/janus/SearchContextCache.java | 4 ++++ .../src/main/java/org/apache/atlas/RequestContext.java | 2 +- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistry.java b/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistry.java index ef8a49951cb..89741554872 100644 --- a/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistry.java +++ b/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistry.java @@ -10,7 +10,7 @@ public interface MetricsRegistry { void collect(String requestId, String requestUri, AtlasPerfMetrics metrics); - void collectIApplicationMetrics(String requestId, String requestUri, List applicationMetrics); + void collectApplicationMetrics(String requestId, String requestUri, List applicationMetrics); void scrape(PrintWriter writer) throws IOException; diff --git a/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistryServiceImpl.java b/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistryServiceImpl.java index e732ad13f0d..19171325e27 100644 --- a/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistryServiceImpl.java +++ b/common/src/main/java/org/apache/atlas/service/metrics/MetricsRegistryServiceImpl.java @@ -62,7 +62,7 @@ public void collect(String requestId, String requestUri, AtlasPerfMetrics metric } } //Use this if you want to publish Histograms - public void collectIApplicationMetrics(String requestId, String requestUri, List applicationMetrics){ + public void collectApplicationMetrics(String requestId, String requestUri, List applicationMetrics){ try { for(AtlasPerfMetrics.Metric metric : applicationMetrics){ if (metric.getMetricType() == AtlasMetricType.COUNTER) { diff --git a/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java b/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java index b259c32c16c..efe90aa887b 100644 --- a/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java +++ b/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java @@ -152,7 +152,7 @@ Config getCacheImplConfig() { .addSentinelAddress(formatUrls(atlasConfig.getStringArray(ATLAS_REDIS_SENTINEL_URLS))) .setUsername(atlasConfig.getString(ATLAS_REDIS_USERNAME)) .setPassword(atlasConfig.getString(ATLAS_REDIS_PASSWORD)) - .setTimeout(50) //Setting UP timeout to 10ms + .setTimeout(50) //Setting UP timeout to 50ms .setRetryAttempts(0); return config; } diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java index a291ac40a06..c3030291460 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java @@ -180,11 +180,11 @@ private DirectIndexQueryResult performAsyncDirectIndexQuery(SearchParams searchP boolean contextIdExists = StringUtils.isNotEmpty(searchParams.getSearchContextId()) && searchParams.getSearchContextSequenceNo() != null; try { if(contextIdExists) { - // If the search context id and greater sequence no is present, then we need to delete the previous search context async + // If the search context id and greater sequence no is present, + // then we need to delete the previous search context async processRequestWithSameSearchContextId(searchParams); } AsyncQueryResult response = submitAsyncSearch(searchParams, false).get(); - //Sleep for 5 seconds to allow ES to process the request if(response.isRunning()) { /* * If the response is still running, then we need to wait for the response @@ -196,11 +196,7 @@ private DirectIndexQueryResult performAsyncDirectIndexQuery(SearchParams searchP String searchContextId = searchParams.getSearchContextId(); Integer searchContextSequenceNo = searchParams.getSearchContextSequenceNo(); if (contextIdExists) { - try { - CompletableFuture.runAsync(() -> SearchContextCache.put(searchContextId, searchContextSequenceNo, esSearchId)); - } catch (Exception e) { - LOG.error("Failed to update the search context cache {}", e.getMessage()); - } + CompletableFuture.runAsync(() -> SearchContextCache.put(searchContextId, searchContextSequenceNo, esSearchId)); } response = getAsyncSearchResponse(searchParams, esSearchId).get(); if (response == null) { diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/SearchContextCache.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/SearchContextCache.java index 0b80ce03344..06937f4646b 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/SearchContextCache.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/SearchContextCache.java @@ -2,10 +2,13 @@ import org.apache.atlas.RequestContext; import org.apache.atlas.service.redis.RedisService; import org.apache.atlas.utils.AtlasPerfMetrics; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Qualifier; import org.springframework.stereotype.Component; @Component public class SearchContextCache { + private static final Logger LOG = LoggerFactory.getLogger(SearchContextCache.class); private static RedisService redisService = null; public static final String INVALID_SEQUENCE = "invalid_sequence"; @@ -30,6 +33,7 @@ public static String get(String key) { try { return redisService.getValue(key); } catch (Exception e) { + LOG.error("Error while fetching value from Redis", e); return null; } diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java b/server-api/src/main/java/org/apache/atlas/RequestContext.java index 9fe4097f5f3..565832b7bd5 100644 --- a/server-api/src/main/java/org/apache/atlas/RequestContext.java +++ b/server-api/src/main/java/org/apache/atlas/RequestContext.java @@ -177,7 +177,7 @@ public void clearCache() { } if (CollectionUtils.isNotEmpty(applicationMetrics)) { if (Objects.nonNull(this.metricsRegistry)){ - this.metricsRegistry.collectIApplicationMetrics(traceId, this.requestUri, applicationMetrics); + this.metricsRegistry.collectApplicationMetrics(traceId, this.requestUri, applicationMetrics); } applicationMetrics.clear(); } From ef47796db37fbb6188b57162d8cb2854a517f579 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Fri, 3 May 2024 12:54:16 +0530 Subject: [PATCH 113/316] Force serialisation of contract json to keep the ordering --- .../v2/preprocessor/contract/ContractPreProcessor.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index df3efde85f6..bc9ff76cbd8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -114,10 +114,9 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con authorizeContractCreateOrUpdate(entity, associatedAsset); boolean contractSync = syncContractCertificateStatus(entity, contract); - if (contractSync) { - contractString = DataContract.serialize(contract); - entity.setAttribute(ATTR_CONTRACT, contractString); - } + contractString = DataContract.serialize(contract); + entity.setAttribute(ATTR_CONTRACT, contractString); + AtlasEntity currentVersionEntity = getCurrentVersion(associatedAsset.getEntity().getGuid()); Long newVersionNumber = 1L; From e5bbaf9fe01c9e4cbbd8fc27fa841e15c9b17a77 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Fri, 3 May 2024 13:10:01 +0530 Subject: [PATCH 114/316] DG-996 Code dedup --- .../datamesh/AbstractDomainPreProcessor.java | 32 +++++++++++++ .../datamesh/DataProductPreProcessor.java | 45 ++----------------- .../datamesh/DomainPreProcessor.java | 21 ++------- 3 files changed, 38 insertions(+), 60 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index a9efa5e98f6..7636f9c0b76 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -37,6 +37,8 @@ import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; +import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -194,4 +196,34 @@ protected void recordUpdatedChildEntities(AtlasVertex entityVertex, Map attributes) throws AtlasBaseException { + if (parentObject == null) { + return null; + } + + AtlasObjectId objectId; + if (parentObject instanceof Map) { + objectId = getAtlasObjectIdFromMapObject(parentObject); + } else { + objectId = (AtlasObjectId) parentObject; + } + + AtlasVertex parentVertex = entityRetriever.getEntityVertex(objectId); + return entityRetriever.toAtlasEntityHeader(parentVertex, attributes); + } + + public static AtlasObjectId getAtlasObjectIdFromMapObject(Object obj) { + Map parentMap = (Map) obj; + AtlasObjectId objectId = new AtlasObjectId(); + objectId.setTypeName((String) parentMap.get("typeName")); + + if (parentMap.containsKey("guid")) { + objectId.setGuid((String) parentMap.get("guid")); + } else { + objectId.setUniqueAttributes((Map) parentMap.get("uniqueAttributes")); + } + + return objectId; + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 4f5681c072e..fd407ffed67 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -48,7 +48,7 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co AtlasEntity entity = (AtlasEntity) entityStruct; AtlasVertex vertex = context.getVertex(entity.getGuid()); - setParent(entity, context); + setParent(entity); switch (operation) { case CREATE: @@ -160,52 +160,13 @@ private void processMoveDataProductToAnotherDomain(AtlasEntity product, } } - private void setParent(AtlasEntity entity, EntityMutationContext context) throws AtlasBaseException { + private void setParent(AtlasEntity entity) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("DataProductPreProcessor.setParent"); if (parentDomain == null) { Object relationshipAttribute = entity.getRelationshipAttribute(DATA_DOMAIN); Set attributes = new HashSet<>(Arrays.asList(QUALIFIED_NAME, SUPER_DOMAIN_QN, PARENT_DOMAIN_QN, "__typeName")); - if(relationshipAttribute instanceof AtlasObjectId){ - AtlasObjectId objectId = (AtlasObjectId) relationshipAttribute; - if (objectId != null) { - if (StringUtils.isNotEmpty(objectId.getGuid())) { - AtlasVertex vertex = entityRetriever.getEntityVertex(objectId.getGuid()); - - if (vertex == null) { - parentDomain = entityRetriever.toAtlasEntityHeader(objectId.getGuid(), attributes); - } else { - parentDomain = entityRetriever.toAtlasEntityHeader(vertex, attributes); - } - - } else if (MapUtils.isNotEmpty(objectId.getUniqueAttributes()) && - StringUtils.isNotEmpty((String) objectId.getUniqueAttributes().get(QUALIFIED_NAME))) { - AtlasVertex parentDomainVertex = entityRetriever.getEntityVertex(objectId); - parentDomain = entityRetriever.toAtlasEntityHeader(parentDomainVertex, attributes); - - } - } - } - else if(relationshipAttribute instanceof Map){ - Map relationshipMap = (Map) relationshipAttribute; - if (StringUtils.isNotEmpty((String) relationshipMap.get("guid"))) { - AtlasVertex vertex = entityRetriever.getEntityVertex((String) relationshipMap.get("guid")); - - if (vertex == null) { - parentDomain = entityRetriever.toAtlasEntityHeader((String) relationshipMap.get("guid"), attributes); - } else { - parentDomain = entityRetriever.toAtlasEntityHeader(vertex, attributes); - } - - } - else { - parentDomain = new AtlasEntityHeader((String) relationshipMap.get("typeName"), relationshipMap); - - } - } - else{ - LOG.warn("DataProductPreProcessor.setParent: Invalid relationshipAttribute {}", relationshipAttribute); - } + parentDomain = getParent(relationshipAttribute, attributes); } RequestContext.get().endMetricRecord(metricRecorder); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index 7ba2addc3fb..91d24284400 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -75,7 +75,7 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co AtlasEntity entity = (AtlasEntity) entityStruct; AtlasVertex vertex = context.getVertex(entity.getGuid()); - setParent(entity, context); + setParent(entity); switch (operation) { case CREATE: @@ -289,27 +289,12 @@ private void moveChildDataProductToAnotherDomain(AtlasVertex productVertex, } } - private void setParent(AtlasEntity entity, EntityMutationContext context) throws AtlasBaseException { + private void setParent(AtlasEntity entity) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("DomainPreProcessor.setParent"); if (parentDomain == null) { AtlasObjectId objectId = (AtlasObjectId) entity.getRelationshipAttribute(PARENT_DOMAIN); Set attributes = new HashSet<>(Arrays.asList(QUALIFIED_NAME, SUPER_DOMAIN_QN, PARENT_DOMAIN_QN, "__typeName")); - - if (objectId != null) { - if (StringUtils.isNotEmpty(objectId.getGuid())) { - AtlasVertex vertex = entityRetriever.getEntityVertex(objectId.getGuid()); - - if (vertex == null) { - parentDomain = entityRetriever.toAtlasEntityHeader(objectId.getGuid(), attributes); - } else { - parentDomain = entityRetriever.toAtlasEntityHeader(vertex, attributes); - } - } else if (MapUtils.isNotEmpty(objectId.getUniqueAttributes()) && - StringUtils.isNotEmpty((String) objectId.getUniqueAttributes().get(QUALIFIED_NAME))) { - AtlasVertex parentDomainVertex = entityRetriever.getEntityVertex(objectId); - parentDomain = entityRetriever.toAtlasEntityHeader(parentDomainVertex, attributes); - } - } + parentDomain = getParent(objectId, attributes); } RequestContext.get().endMetricRecord(metricRecorder); } From 34acadc9ef05a18fe0269a337d9ab24c45abfc4f Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Fri, 3 May 2024 13:53:14 +0530 Subject: [PATCH 115/316] resolved case 4 --- .../datamesh/AbstractDomainPreProcessor.java | 1 + .../preprocessor/datamesh/DataProductPreProcessor.java | 10 ++++++---- .../v2/preprocessor/datamesh/DomainPreProcessor.java | 8 +++++--- 3 files changed, 12 insertions(+), 7 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 7636f9c0b76..72f27eb679a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -107,6 +107,7 @@ protected void updatePolicy(List currentResources, Map u AtlasEntityType entityType = typeRegistry.getEntityTypeByName(POLICY_ENTITY_TYPE); for (AtlasEntityHeader policy : policies) { + LOG.info("Updating policy {}", policy); AtlasVertex policyVertex = entityRetriever.getEntityVertex(policy.getGuid()); AtlasEntity policyEntity = entityRetriever.toAtlasEntity(policyVertex); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index fd407ffed67..9d0a43ae057 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -128,6 +128,7 @@ private void processMoveDataProductToAnotherDomain(AtlasEntity product, try { String productName = (String) product.getAttribute(NAME); + String updatedQualifiedName; if(StringUtils.isEmpty(targetDomainQualifiedName)){ throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "DataProduct can only be moved to another Domain."); @@ -138,10 +139,11 @@ private void processMoveDataProductToAnotherDomain(AtlasEntity product, productExists(productName, targetDomainQualifiedName); if(StringUtils.isEmpty(sourceDomainQualifiedName)){ - sourceDomainQualifiedName = "default"; + updatedQualifiedName = targetDomainQualifiedName + product.getAttribute(QUALIFIED_NAME); + } + else{ + updatedQualifiedName = currentDataProductQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); } - - String updatedQualifiedName = currentDataProductQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); product.setAttribute(QUALIFIED_NAME, updatedQualifiedName); product.setAttribute(PARENT_DOMAIN_QN, targetDomainQualifiedName); @@ -184,7 +186,7 @@ private void productExists(String productName, String parentDomainQualifiedName) Map bool = new HashMap<>(); - if (parentDomain != null) { + if (parentDomain != null && StringUtils.isNotEmpty(parentDomainQualifiedName)){ mustClauseList.add(mapOf("term", mapOf("parentDomainQualifiedName", parentDomainQualifiedName))); } else { List> mustNotClauseList = new ArrayList(); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index 91d24284400..257b368ca1c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -90,9 +90,11 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co private void processCreateDomain(AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreateDomain"); String domainName = (String) entity.getAttribute(NAME); - String parentDomainQualifiedName = (String) entity.getAttribute(PARENT_DOMAIN_QN); + if(parentDomain != null ){ + String parentDomainQualifiedName = (String) parentDomain.getAttribute(QUALIFIED_NAME); + domainExists(domainName, parentDomainQualifiedName); + } - domainExists(domainName, parentDomainQualifiedName); RequestContext.get().endMetricRecord(metricRecorder); } @@ -311,7 +313,7 @@ private void domainExists(String domainName, String parentDomainQualifiedName) t Map bool = new HashMap<>(); - if (parentDomain != null) { + if (parentDomain != null && StringUtils.isNotEmpty(parentDomainQualifiedName)) { mustClauseList.add(mapOf("term", mapOf("parentDomainQualifiedName", parentDomainQualifiedName))); } else { List> mustNotClauseList = new ArrayList(); From 91839c94c11ff8cceedbce2ff519ecab1bb808fa Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Fri, 3 May 2024 14:49:54 +0530 Subject: [PATCH 116/316] resolved same name domain issue --- .../v2/preprocessor/datamesh/DataProductPreProcessor.java | 6 ++++-- .../graph/v2/preprocessor/datamesh/DomainPreProcessor.java | 7 +++---- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 9d0a43ae057..b3afd860df8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -63,9 +63,11 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co private void processCreateProduct(AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreateProduct"); String productName = (String) entity.getAttribute(NAME); - String parentDomainQualifiedName = (String) entity.getAttribute(PARENT_DOMAIN_QN); + if(parentDomain != null ){ + String parentDomainQualifiedName = (String) parentDomain.getAttribute(QUALIFIED_NAME); + productExists(productName, parentDomainQualifiedName); + } - productExists(productName, parentDomainQualifiedName); RequestContext.get().endMetricRecord(metricRecorder); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index 257b368ca1c..b0798b2e62b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -90,10 +90,9 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co private void processCreateDomain(AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreateDomain"); String domainName = (String) entity.getAttribute(NAME); - if(parentDomain != null ){ - String parentDomainQualifiedName = (String) parentDomain.getAttribute(QUALIFIED_NAME); - domainExists(domainName, parentDomainQualifiedName); - } + String parentDomainQualifiedName = (String) entity.getAttribute(PARENT_DOMAIN_QN); + + domainExists(domainName, parentDomainQualifiedName); RequestContext.get().endMetricRecord(metricRecorder); } From 2da649ec3e9eb5e3537c37baec66360aeb4529b3 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Fri, 3 May 2024 15:01:35 +0530 Subject: [PATCH 117/316] same product name issue resolved --- .../v2/preprocessor/datamesh/DataProductPreProcessor.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index b3afd860df8..d14a7778974 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -63,11 +63,13 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co private void processCreateProduct(AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreateProduct"); String productName = (String) entity.getAttribute(NAME); + String parentDomainQualifiedName = (String) entity.getAttribute(PARENT_DOMAIN_QN); if(parentDomain != null ){ - String parentDomainQualifiedName = (String) parentDomain.getAttribute(QUALIFIED_NAME); - productExists(productName, parentDomainQualifiedName); + parentDomainQualifiedName = (String) parentDomain.getAttribute(QUALIFIED_NAME); } + productExists(productName, parentDomainQualifiedName); + RequestContext.get().endMetricRecord(metricRecorder); } From 1b7b5585407d4e0786fa31e3524861070b6c8bc6 Mon Sep 17 00:00:00 2001 From: Rajat Movaliya Date: Fri, 3 May 2024 14:31:00 +0530 Subject: [PATCH 118/316] add separate counter for traversalOrder --- .../atlas/discovery/EntityLineageService.java | 40 ++++++++++--------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index 353232cdbff..5bdecd451fc 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -294,12 +294,13 @@ private AtlasLineageOnDemandInfo getLineageInfoOnDemand(String guid, AtlasLineag AtomicInteger inputEntitiesTraversed = new AtomicInteger(0); AtomicInteger outputEntitiesTraversed = new AtomicInteger(0); + AtomicInteger traversalOrder = new AtomicInteger(1); if (isDataSet) { AtlasVertex datasetVertex = AtlasGraphUtilsV2.findByGuid(this.graph, guid); if (direction == AtlasLineageOnDemandInfo.LineageDirection.INPUT || direction == AtlasLineageOnDemandInfo.LineageDirection.BOTH) - traverseEdgesOnDemand(datasetVertex, true, depth, level, new HashSet<>(), atlasLineageOnDemandContext, ret, guid, inputEntitiesTraversed); + traverseEdgesOnDemand(datasetVertex, true, depth, level, new HashSet<>(), atlasLineageOnDemandContext, ret, guid, inputEntitiesTraversed, traversalOrder); if (direction == AtlasLineageOnDemandInfo.LineageDirection.OUTPUT || direction == AtlasLineageOnDemandInfo.LineageDirection.BOTH) - traverseEdgesOnDemand(datasetVertex, false, depth, level, new HashSet<>(), atlasLineageOnDemandContext, ret, guid, outputEntitiesTraversed); + traverseEdgesOnDemand(datasetVertex, false, depth, level, new HashSet<>(), atlasLineageOnDemandContext, ret, guid, outputEntitiesTraversed, traversalOrder); AtlasEntityHeader baseEntityHeader = entityRetriever.toAtlasEntityHeader(datasetVertex, atlasLineageOnDemandContext.getAttributes()); baseEntityHeader.setDepth(level); ret.getGuidEntityMap().put(guid, baseEntityHeader); @@ -308,19 +309,18 @@ private AtlasLineageOnDemandInfo getLineageInfoOnDemand(String guid, AtlasLineag // make one hop to the next dataset vertices from process vertex and traverse with 'depth = depth - 1' if (direction == AtlasLineageOnDemandInfo.LineageDirection.INPUT || direction == AtlasLineageOnDemandInfo.LineageDirection.BOTH) { Iterator processEdges = processVertex.getEdges(AtlasEdgeDirection.OUT, PROCESS_INPUTS_EDGE).iterator(); - traverseEdgesOnDemand(processEdges, true, depth, level, atlasLineageOnDemandContext, ret, processVertex, guid, inputEntitiesTraversed); + traverseEdgesOnDemand(processEdges, true, depth, level, atlasLineageOnDemandContext, ret, processVertex, guid, inputEntitiesTraversed, traversalOrder); } if (direction == AtlasLineageOnDemandInfo.LineageDirection.OUTPUT || direction == AtlasLineageOnDemandInfo.LineageDirection.BOTH) { Iterator processEdges = processVertex.getEdges(AtlasEdgeDirection.OUT, PROCESS_OUTPUTS_EDGE).iterator(); - traverseEdgesOnDemand(processEdges, false, depth, level, atlasLineageOnDemandContext, ret, processVertex, guid, outputEntitiesTraversed); + traverseEdgesOnDemand(processEdges, false, depth, level, atlasLineageOnDemandContext, ret, processVertex, guid, outputEntitiesTraversed, traversalOrder); } } RequestContext.get().endMetricRecord(metricRecorder); return ret; } - - private void traverseEdgesOnDemand(Iterator processEdges, boolean isInput, int depth, int level, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, AtlasVertex processVertex, String baseGuid, AtomicInteger entitiesTraversed) throws AtlasBaseException { + private void traverseEdgesOnDemand(Iterator processEdges, boolean isInput, int depth, int level, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, AtlasVertex processVertex, String baseGuid, AtomicInteger entitiesTraversed, AtomicInteger traversalOrder) throws AtlasBaseException { AtlasLineageOnDemandInfo.LineageDirection direction = isInput ? AtlasLineageOnDemandInfo.LineageDirection.INPUT : AtlasLineageOnDemandInfo.LineageDirection.OUTPUT; int nextLevel = isInput ? level - 1: level + 1; @@ -340,7 +340,8 @@ private void traverseEdgesOnDemand(Iterator processEdges, boolean isI if (incrementAndCheckIfRelationsLimitReached(processEdge, isInputEdge, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, direction)) { break; } else { - addEdgeToResult(processEdge, ret, atlasLineageOnDemandContext, nextLevel, entitiesTraversed); + addEdgeToResult(processEdge, ret, atlasLineageOnDemandContext, nextLevel, traversalOrder); + traversalOrder.incrementAndGet(); } String inGuid = AtlasGraphUtilsV2.getIdFromVertex(datasetVertex); @@ -350,11 +351,11 @@ private void traverseEdgesOnDemand(Iterator processEdges, boolean isI ret.getRelationsOnDemand().put(inGuid, new LineageInfoOnDemand(inGuidLineageConstrains)); } - traverseEdgesOnDemand(datasetVertex, isInput, depth - 1, nextLevel, new HashSet<>(), atlasLineageOnDemandContext, ret, baseGuid, entitiesTraversed); + traverseEdgesOnDemand(datasetVertex, isInput, depth - 1, nextLevel, new HashSet<>(), atlasLineageOnDemandContext, ret, baseGuid, entitiesTraversed, traversalOrder); } } - private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, int depth, int level, Set visitedVertices, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, String baseGuid, AtomicInteger entitiesTraversed) throws AtlasBaseException { + private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, int depth, int level, Set visitedVertices, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, String baseGuid, AtomicInteger entitiesTraversed, AtomicInteger traversalOrder) throws AtlasBaseException { if (isEntityTraversalLimitReached(entitiesTraversed)) return; if (depth != 0) { // base condition of recursion for depth @@ -389,7 +390,7 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i else continue; } else { - addEdgeToResult(incomingEdge, ret, atlasLineageOnDemandContext, level, entitiesTraversed); + addEdgeToResult(incomingEdge, ret, atlasLineageOnDemandContext, level, traversalOrder); } AtlasPerfMetrics.MetricRecorder traverseEdgesOnDemandGetEdgesOut = RequestContext.get().startMetricRecord("traverseEdgesOnDemandGetEdgesOut"); @@ -417,13 +418,14 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i else continue; } else { - addEdgeToResult(outgoingEdge, ret, atlasLineageOnDemandContext, nextLevel, entitiesTraversed); + addEdgeToResult(outgoingEdge, ret, atlasLineageOnDemandContext, nextLevel, traversalOrder); entitiesTraversed.incrementAndGet(); + traversalOrder.incrementAndGet(); if (isEntityTraversalLimitReached(entitiesTraversed)) setEntityLimitReachedFlag(isInput, ret); } if (entityVertex != null && !visitedVertices.contains(getId(entityVertex))) { - traverseEdgesOnDemand(entityVertex, isInput, depth - 1, nextLevel, visitedVertices, atlasLineageOnDemandContext, ret, baseGuid, entitiesTraversed); // execute inner depth + traverseEdgesOnDemand(entityVertex, isInput, depth - 1, nextLevel, visitedVertices, atlasLineageOnDemandContext, ret, baseGuid, entitiesTraversed, traversalOrder); // execute inner depth } } } @@ -878,9 +880,9 @@ private void addEdgeToResult(AtlasEdge edge, AtlasLineageInfo lineageInfo, } } - private void addEdgeToResult(AtlasEdge edge, AtlasLineageOnDemandInfo lineageInfo, AtlasLineageOnDemandContext atlasLineageOnDemandContext, int level, AtomicInteger entitiesTraversed) throws AtlasBaseException { + private void addEdgeToResult(AtlasEdge edge, AtlasLineageOnDemandInfo lineageInfo, AtlasLineageOnDemandContext atlasLineageOnDemandContext, int level, AtomicInteger traversalOrder) throws AtlasBaseException { if (!lineageContainsVisitedEdgeV2(lineageInfo, edge)) { - processEdge(edge, lineageInfo, atlasLineageOnDemandContext, level, entitiesTraversed); + processEdge(edge, lineageInfo, atlasLineageOnDemandContext, level, traversalOrder); } } @@ -1450,11 +1452,11 @@ private void processEdge(final AtlasEdge edge, final Map entities, final Set relations, final Set visitedEdges, final Set attributes, int level, AtomicInteger entitiesTraversed) throws AtlasBaseException { + private void processEdge(final AtlasEdge edge, final Map entities, final Set relations, final Set visitedEdges, final Set attributes, int level, AtomicInteger traversalOrder) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processEdge"); AtlasVertex inVertex = edge.getInVertex(); AtlasVertex outVertex = edge.getOutVertex(); @@ -1468,7 +1470,7 @@ private void processEdge(final AtlasEdge edge, final Map Date: Thu, 2 May 2024 22:35:41 +0530 Subject: [PATCH 119/316] feat: add feature flag --- .../graphdb/janus/FeatureFlagStore.java | 30 +++++++++++++++++++ pom.xml | 2 +- .../atlas/web/filters/ActiveServerFilter.java | 17 ++++++----- 3 files changed, 41 insertions(+), 8 deletions(-) create mode 100644 graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/FeatureFlagStore.java diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/FeatureFlagStore.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/FeatureFlagStore.java new file mode 100644 index 00000000000..1b37f7cf0ca --- /dev/null +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/FeatureFlagStore.java @@ -0,0 +1,30 @@ +package org.apache.atlas.repository.graphdb.janus; + +import org.apache.atlas.service.redis.RedisService; +import org.apache.commons.lang.StringUtils; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.stereotype.Component; + +@Component +public class FeatureFlagStore { + private static RedisService redisService = null; + public FeatureFlagStore(@Qualifier("redisServiceImpl") RedisService redisService) { + FeatureFlagStore.redisService = redisService; + } + + public static boolean evaluate(String key,String expectedValue) { + String updatedKey = createFeatureFlagNamespace(key); + if (updatedKey == null) { + return false; + } + String value = redisService.getValue(updatedKey); + return StringUtils.equals(value, expectedValue); + } + + private static String createFeatureFlagNamespace(String value) { + if(StringUtils.isEmpty(value)) { + return null; + } + return "ff:"+ value; + } +} diff --git a/pom.xml b/pom.xml index 1cc9aa70dc9..3abc39041b7 100644 --- a/pom.xml +++ b/pom.xml @@ -717,7 +717,7 @@ 4.4.13 2.12.4 2.12.4 - 0.6.03 + 0.6.04 0.5.3 1 3.1.0 diff --git a/webapp/src/main/java/org/apache/atlas/web/filters/ActiveServerFilter.java b/webapp/src/main/java/org/apache/atlas/web/filters/ActiveServerFilter.java index cedf9c201a2..4798cbe0a8d 100644 --- a/webapp/src/main/java/org/apache/atlas/web/filters/ActiveServerFilter.java +++ b/webapp/src/main/java/org/apache/atlas/web/filters/ActiveServerFilter.java @@ -21,6 +21,7 @@ import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.repository.graphdb.janus.FeatureFlagStore; import org.apache.atlas.type.AtlasType; import org.apache.atlas.web.service.ActiveInstanceState; import org.apache.atlas.web.service.ServiceState; @@ -88,13 +89,15 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo FilterChain filterChain) throws IOException, ServletException { // If maintenance mode is enabled, return a 503 if (AtlasConfiguration.ATLAS_MAINTENANCE_MODE.getBoolean()) { - // Block all the POST, PUT, DELETE operations - HttpServletRequest request = (HttpServletRequest) servletRequest; - HttpServletResponse response = (HttpServletResponse) servletResponse; - if (isBlockedMethod(request.getMethod()) && !isWhitelistedAPI(request.getRequestURI())) { - LOG.error("Maintenance mode enabled. Blocking request: {}", request.getRequestURI()); - sendMaintenanceModeResponse(response); - return; // Stop further processing + if (FeatureFlagStore.evaluate("maintenance_mode", "true")) { + // Block all the POST, PUT, DELETE operations + HttpServletRequest request = (HttpServletRequest) servletRequest; + HttpServletResponse response = (HttpServletResponse) servletResponse; + if (isBlockedMethod(request.getMethod()) && !isWhitelistedAPI(request.getRequestURI())) { + LOG.error("Maintenance mode enabled. Blocking request: {}", request.getRequestURI()); + sendMaintenanceModeResponse(response); + return; // Stop further processing + } } } From db3fbb7a16c0975075f75eeeb34f9ee4ab310b51 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Fri, 3 May 2024 19:04:33 +0530 Subject: [PATCH 120/316] DG-996 Fix a few edge cases --- .../v2/preprocessor/datamesh/DataProductPreProcessor.java | 2 +- .../graph/v2/preprocessor/datamesh/DomainPreProcessor.java | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index d14a7778974..8eaadca1f25 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -143,7 +143,7 @@ private void processMoveDataProductToAnotherDomain(AtlasEntity product, productExists(productName, targetDomainQualifiedName); if(StringUtils.isEmpty(sourceDomainQualifiedName)){ - updatedQualifiedName = targetDomainQualifiedName + product.getAttribute(QUALIFIED_NAME); + updatedQualifiedName = targetDomainQualifiedName + "/" + product.getAttribute(QUALIFIED_NAME); } else{ updatedQualifiedName = currentDataProductQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index b0798b2e62b..d210a7420b2 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -91,6 +91,9 @@ private void processCreateDomain(AtlasEntity entity, AtlasVertex vertex) throws AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreateDomain"); String domainName = (String) entity.getAttribute(NAME); String parentDomainQualifiedName = (String) entity.getAttribute(PARENT_DOMAIN_QN); + if(parentDomain != null ){ + parentDomainQualifiedName = (String) parentDomain.getAttribute(QUALIFIED_NAME); + } domainExists(domainName, parentDomainQualifiedName); From e77f97d584e239f9f49db189e2ad984cc4374c63 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Fri, 3 May 2024 19:19:58 +0530 Subject: [PATCH 121/316] poliCategory check --- .../store/graph/v2/preprocessor/PreProcessorUtils.java | 1 + .../v2/preprocessor/datamesh/AbstractDomainPreProcessor.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index d1f76296bd4..14dcff2665a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -49,6 +49,7 @@ public class PreProcessorUtils { public static final String PARENT_DOMAIN_QN = "parentDomainQualifiedName"; public static final String SUPER_DOMAIN_QN = "superDomainQualifiedName"; public static final String DATA_DOMAIN = "dataDomain"; + public static final String MESH_POLICY_CATEGORY = "datamesh"; public static final String MIGRATION_CUSTOM_ATTRIBUTE = "isQualifiedNameMigrated"; //Query models constants diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 72f27eb679a..b560c0b2340 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -112,7 +112,7 @@ protected void updatePolicy(List currentResources, Map u AtlasEntity policyEntity = entityRetriever.toAtlasEntity(policyVertex); - if (policyEntity.hasRelationshipAttribute("accessControl")) { + if (policyEntity.hasRelationshipAttribute("accessControl") && policyEntity.getAttribute(ATTR_POLICY_CATEGORY) != MESH_POLICY_CATEGORY) { AtlasVertex accessControl = entityRetriever.getEntityVertex(((AtlasObjectId) policyEntity.getRelationshipAttribute("accessControl")).getGuid()); context.getDiscoveryContext().addResolvedGuid(GraphHelper.getGuid(accessControl), accessControl); } From 8d25b57053f859d8a4864c33acc47b819572e957 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Fri, 3 May 2024 19:50:10 +0530 Subject: [PATCH 122/316] feat: add feature flag feature in Atlas --- .../service/redis/AbstractRedisService.java | 11 +++++- .../service/redis/NoRedisServiceImpl.java | 2 +- .../atlas/service/redis/RedisService.java | 2 + .../service/redis/RedisServiceLocalImpl.java | 2 +- .../graphdb/janus/FeatureFlagStore.java | 37 +++++++++++++------ .../graphdb/janus/SearchContextCache.java | 2 +- .../org/apache/atlas/AtlasConfiguration.java | 2 +- pom.xml | 2 +- .../atlas/web/filters/ActiveServerFilter.java | 4 +- .../atlas/web/resources/AdminResource.java | 14 +++++++ 10 files changed, 58 insertions(+), 20 deletions(-) diff --git a/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java b/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java index efe90aa887b..77ba5e614f8 100644 --- a/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java +++ b/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java @@ -81,7 +81,14 @@ public String getValue(String key) { @Override public String putValue(String key, String value) { // Put the value in the redis cache with TTL - redisCacheClient.getBucket(convertToNamespace(key)).set(value, 30, TimeUnit.SECONDS); + redisCacheClient.getBucket(convertToNamespace(key)).set(value); + return value; + } + + @Override + public String putValue(String key, String value, int timeout) { + // Put the value in the redis cache with TTL + redisCacheClient.getBucket(convertToNamespace(key)).set(value, timeout, TimeUnit.SECONDS); return value; } @@ -152,7 +159,7 @@ Config getCacheImplConfig() { .addSentinelAddress(formatUrls(atlasConfig.getStringArray(ATLAS_REDIS_SENTINEL_URLS))) .setUsername(atlasConfig.getString(ATLAS_REDIS_USERNAME)) .setPassword(atlasConfig.getString(ATLAS_REDIS_PASSWORD)) - .setTimeout(50) //Setting UP timeout to 50ms + .setTimeout(500) //Setting UP timeout to 50ms .setRetryAttempts(0); return config; } diff --git a/common/src/main/java/org/apache/atlas/service/redis/NoRedisServiceImpl.java b/common/src/main/java/org/apache/atlas/service/redis/NoRedisServiceImpl.java index 5bc089d9557..9bd942d3042 100644 --- a/common/src/main/java/org/apache/atlas/service/redis/NoRedisServiceImpl.java +++ b/common/src/main/java/org/apache/atlas/service/redis/NoRedisServiceImpl.java @@ -35,7 +35,7 @@ public String getValue(String key) { } @Override - public String putValue(String key, String value) { + public String putValue(String key, String value, int timeout) { return null; } diff --git a/common/src/main/java/org/apache/atlas/service/redis/RedisService.java b/common/src/main/java/org/apache/atlas/service/redis/RedisService.java index 6cb2f04ada3..a541b1eeeef 100644 --- a/common/src/main/java/org/apache/atlas/service/redis/RedisService.java +++ b/common/src/main/java/org/apache/atlas/service/redis/RedisService.java @@ -12,6 +12,8 @@ public interface RedisService { String putValue(String key, String value); + String putValue(String key, String value, int timeout); + void removeValue(String key); Logger getLogger(); diff --git a/common/src/main/java/org/apache/atlas/service/redis/RedisServiceLocalImpl.java b/common/src/main/java/org/apache/atlas/service/redis/RedisServiceLocalImpl.java index c625f105f13..c69a151a7d2 100644 --- a/common/src/main/java/org/apache/atlas/service/redis/RedisServiceLocalImpl.java +++ b/common/src/main/java/org/apache/atlas/service/redis/RedisServiceLocalImpl.java @@ -28,7 +28,7 @@ public String getValue(String key) { } @Override - public String putValue(String key, String value) { + public String putValue(String key, String value, int timeout) { return null; } diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/FeatureFlagStore.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/FeatureFlagStore.java index 1b37f7cf0ca..87b68635efb 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/FeatureFlagStore.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/FeatureFlagStore.java @@ -12,19 +12,34 @@ public FeatureFlagStore(@Qualifier("redisServiceImpl") RedisService redisService FeatureFlagStore.redisService = redisService; } - public static boolean evaluate(String key,String expectedValue) { - String updatedKey = createFeatureFlagNamespace(key); - if (updatedKey == null) { - return false; + public static boolean evaluate(String key, String expectedValue) { + boolean ret = false; + try{ + if (StringUtils.isEmpty(key) || StringUtils.isEmpty(expectedValue)) + return ret; + String value = redisService.getValue(addFeatureFlagNamespace(key)); + ret = StringUtils.equals(value, expectedValue); + } catch (Exception e) { + return ret; } - String value = redisService.getValue(updatedKey); - return StringUtils.equals(value, expectedValue); + return ret; } - private static String createFeatureFlagNamespace(String value) { - if(StringUtils.isEmpty(value)) { - return null; - } - return "ff:"+ value; + public static void setFlag(String key, String value) { + if (StringUtils.isEmpty(key) || StringUtils.isEmpty(value)) + return; + + redisService.putValue(addFeatureFlagNamespace(key), value); + } + + public static void deleteFlag(String key) { + if (StringUtils.isEmpty(key)) + return; + + redisService.removeValue(addFeatureFlagNamespace(key)); + } + + private static String addFeatureFlagNamespace(String key) { + return "ff:"+key; } } diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/SearchContextCache.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/SearchContextCache.java index 06937f4646b..f7e5718f191 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/SearchContextCache.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/SearchContextCache.java @@ -24,7 +24,7 @@ public static void put(String key, Integer sequence, String esAsyncId) { try { // Build the string in format `sequence/esAsyncId` and store it in redis String val = sequence + "/" + esAsyncId; - redisService.putValue(key, val); + redisService.putValue(key, val, 30); } finally { RequestContext.get().endMetricRecord(metric); } diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index a701510a4e0..02adc39be61 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -113,7 +113,7 @@ public enum AtlasConfiguration { INDEXSEARCH_ASYNC_SEARCH_KEEP_ALIVE_TIME_IN_SECONDS("atlas.indexsearch.async.search.keep.alive.time.in.seconds", 300), - ATLAS_MAINTENANCE_MODE("atlas.maintenance.mode", false); + ATLAS_MAINTENANCE_MODE("atlas.maintenance.mode", true); private static final Configuration APPLICATION_PROPERTIES; diff --git a/pom.xml b/pom.xml index 3abc39041b7..1cc9aa70dc9 100644 --- a/pom.xml +++ b/pom.xml @@ -717,7 +717,7 @@ 4.4.13 2.12.4 2.12.4 - 0.6.04 + 0.6.03 0.5.3 1 3.1.0 diff --git a/webapp/src/main/java/org/apache/atlas/web/filters/ActiveServerFilter.java b/webapp/src/main/java/org/apache/atlas/web/filters/ActiveServerFilter.java index 4798cbe0a8d..1abdd256664 100644 --- a/webapp/src/main/java/org/apache/atlas/web/filters/ActiveServerFilter.java +++ b/webapp/src/main/java/org/apache/atlas/web/filters/ActiveServerFilter.java @@ -57,9 +57,9 @@ public class ActiveServerFilter implements Filter { private static final Logger LOG = LoggerFactory.getLogger(ActiveServerFilter.class); private static final String MIGRATION_STATUS_STATIC_PAGE = "migration-status.html"; - private static final String[] WHITELISTED_APIS_SIGNATURE = {"search", "lineage", "auditSearch", "accessors" , "evaluator"}; + private static final String DISABLE_WRITE_FLAG = "disable_writes"; private final ActiveInstanceState activeInstanceState; private ServiceState serviceState; @@ -89,7 +89,7 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo FilterChain filterChain) throws IOException, ServletException { // If maintenance mode is enabled, return a 503 if (AtlasConfiguration.ATLAS_MAINTENANCE_MODE.getBoolean()) { - if (FeatureFlagStore.evaluate("maintenance_mode", "true")) { + if (FeatureFlagStore.evaluate(DISABLE_WRITE_FLAG, "true")) { // Block all the POST, PUT, DELETE operations HttpServletRequest request = (HttpServletRequest) servletRequest; HttpServletResponse response = (HttpServletResponse) servletResponse; diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java index 3afd2b451d3..c8715a38680 100755 --- a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java +++ b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java @@ -48,6 +48,7 @@ import org.apache.atlas.model.patches.AtlasPatch.AtlasPatches; import org.apache.atlas.model.tasks.AtlasTask; import org.apache.atlas.repository.audit.AtlasAuditService; +import org.apache.atlas.repository.graphdb.janus.FeatureFlagStore; import org.apache.atlas.repository.impexp.AtlasServerService; import org.apache.atlas.repository.impexp.ExportImportAuditService; import org.apache.atlas.repository.impexp.ExportService; @@ -930,6 +931,19 @@ public Map getDebugMetrics() { return debugMetricsRESTSink.getMetrics(); } + @POST + @Path("featureFlag") + @Produces(MediaType.APPLICATION_JSON) + public void setFeatureFlag(@QueryParam("key") String key, @QueryParam("value") String value) { + FeatureFlagStore.setFlag(key, value); + } + + @DELETE + @Path("featureFlag/{flag}") + @Produces(MediaType.APPLICATION_JSON) + public void deleteFeatureFlag(@PathParam("flag") String key) { + FeatureFlagStore.deleteFlag(key); + } private String getEditableEntityTypes(Configuration config) { String ret = DEFAULT_EDITABLE_ENTITY_TYPES; From b8a0f44d45763bf96dc296973a9cbbf2c82e9a45 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Fri, 3 May 2024 20:33:34 +0530 Subject: [PATCH 123/316] feat: set back timeout to 50ms --- .../org/apache/atlas/service/redis/AbstractRedisService.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java b/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java index 77ba5e614f8..9ad5fd904f2 100644 --- a/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java +++ b/common/src/main/java/org/apache/atlas/service/redis/AbstractRedisService.java @@ -159,7 +159,7 @@ Config getCacheImplConfig() { .addSentinelAddress(formatUrls(atlasConfig.getStringArray(ATLAS_REDIS_SENTINEL_URLS))) .setUsername(atlasConfig.getString(ATLAS_REDIS_USERNAME)) .setPassword(atlasConfig.getString(ATLAS_REDIS_PASSWORD)) - .setTimeout(500) //Setting UP timeout to 50ms + .setTimeout(50) //Setting UP timeout to 50ms .setRetryAttempts(0); return config; } From 72b2ff5b51c45bf3abecc0c93ae01d5d1322753b Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Fri, 3 May 2024 22:01:26 +0530 Subject: [PATCH 124/316] debug logs --- .../v2/preprocessor/datamesh/AbstractDomainPreProcessor.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index b560c0b2340..5aec65b2f67 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -111,8 +111,10 @@ protected void updatePolicy(List currentResources, Map u AtlasVertex policyVertex = entityRetriever.getEntityVertex(policy.getGuid()); AtlasEntity policyEntity = entityRetriever.toAtlasEntity(policyVertex); + String policyCategory = (String) policyEntity.getAttribute(ATTR_POLICY_CATEGORY); - if (policyEntity.hasRelationshipAttribute("accessControl") && policyEntity.getAttribute(ATTR_POLICY_CATEGORY) != MESH_POLICY_CATEGORY) { + if (policyEntity.hasRelationshipAttribute("accessControl") && !StringUtils.equals(policyCategory, MESH_POLICY_CATEGORY)) { + LOG.info("PolicyCategory {}", policyCategory); AtlasVertex accessControl = entityRetriever.getEntityVertex(((AtlasObjectId) policyEntity.getRelationshipAttribute("accessControl")).getGuid()); context.getDiscoveryContext().addResolvedGuid(GraphHelper.getGuid(accessControl), accessControl); } From 27b865dc49032643108ccc9bc5021831d49f8117 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Fri, 3 May 2024 23:06:17 +0530 Subject: [PATCH 125/316] call to AuthPolicy pre-processor --- .../repository/store/graph/v2/AtlasEntityStoreV2.java | 4 ++-- .../datamesh/AbstractDomainPreProcessor.java | 11 +++++++++-- .../datamesh/DataProductPreProcessor.java | 5 +++-- .../v2/preprocessor/datamesh/DomainPreProcessor.java | 5 +++-- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 0865eb47074..2e51f45db59 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -1803,11 +1803,11 @@ public PreProcessor getPreProcessor(String typeName) { break; case DATA_DOMAIN_ENTITY_TYPE: - preProcessor = new DomainPreProcessor(typeRegistry, entityRetriever, graph); + preProcessor = new DomainPreProcessor(typeRegistry, entityRetriever, graph, featureFlagStore); break; case DATA_PRODUCT_ENTITY_TYPE: - preProcessor = new DataProductPreProcessor(typeRegistry, entityRetriever, graph); + preProcessor = new DataProductPreProcessor(typeRegistry, entityRetriever, graph, featureFlagStore); break; case QUERY_ENTITY_TYPE: diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 5aec65b2f67..61cf54fa3cf 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -24,14 +24,17 @@ import org.apache.atlas.authorize.AtlasPrivilege; import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.featureflag.FeatureFlagStore; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasObjectId; +import org.apache.atlas.model.instance.EntityMutations; import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; +import org.apache.atlas.repository.store.graph.v2.preprocessor.AuthPolicyPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; @@ -56,12 +59,15 @@ public abstract class AbstractDomainPreProcessor implements PreProcessor { protected final AtlasTypeRegistry typeRegistry; protected final EntityGraphRetriever entityRetriever; - + private final PreProcessor preProcessor; + private final FeatureFlagStore featureFlagStore; protected EntityDiscoveryService discovery; - AbstractDomainPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, AtlasGraph graph) { + AbstractDomainPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, AtlasGraph graph, FeatureFlagStore featureFlagStore) { this.entityRetriever = entityRetriever; this.typeRegistry = typeRegistry; + this.preProcessor = new AuthPolicyPreProcessor(graph, typeRegistry, entityRetriever, featureFlagStore); + this.featureFlagStore = featureFlagStore; try { this.discovery = new EntityDiscoveryService(typeRegistry, graph, null, null, null, null); @@ -134,6 +140,7 @@ protected void updatePolicy(List currentResources, Map u policyVertex.removeProperty(ATTR_POLICY_RESOURCES); policyEntity.setAttribute(ATTR_POLICY_RESOURCES, updatedPolicyResourcesList); context.addUpdated(policyEntity.getGuid(), policyEntity, entityType, policyVertex); + this.preProcessor.processAttributes(policyEntity, context, EntityMutations.EntityOperation.UPDATE); } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 8eaadca1f25..02cc33bb19b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -3,6 +3,7 @@ import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.RequestContext; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.featureflag.FeatureFlagStore; import org.apache.atlas.model.instance.*; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; @@ -29,8 +30,8 @@ public class DataProductPreProcessor extends AbstractDomainPreProcessor { private List currentResources; private Map updatedPolicyResources; public DataProductPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, - AtlasGraph graph) { - super(typeRegistry, entityRetriever, graph); + AtlasGraph graph, FeatureFlagStore featureFlagStore) { + super(typeRegistry, entityRetriever, graph, featureFlagStore); this.currentResources = new ArrayList<>(); this.updatedPolicyResources = new HashMap<>(); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index d210a7420b2..4efce96a0e5 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -21,6 +21,7 @@ import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.RequestContext; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.featureflag.FeatureFlagStore; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasObjectId; @@ -55,8 +56,8 @@ public class DomainPreProcessor extends AbstractDomainPreProcessor { private Map updatedPolicyResources; public DomainPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, - AtlasGraph graph) { - super(typeRegistry, entityRetriever, graph); + AtlasGraph graph, FeatureFlagStore featureFlagStore) { + super(typeRegistry, entityRetriever, graph, featureFlagStore); this.currentResources = new ArrayList<>(); this.updatedPolicyResources = new HashMap<>(); } From d8b32eef6cf1d010109afc6a10896f2aa881c7c7 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Sun, 5 May 2024 10:11:42 +0530 Subject: [PATCH 126/316] audit logs for policies --- .../v2/preprocessor/datamesh/AbstractDomainPreProcessor.java | 3 +++ .../graph/v2/preprocessor/datamesh/DomainPreProcessor.java | 1 + 2 files changed, 4 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 61cf54fa3cf..8487715813f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -107,6 +107,7 @@ protected void updatePolicy(List currentResources, Map u AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("updateDomainPolicy"); try { LOG.info("Updating policies for entities {}", currentResources); + Map updatedAttributes = new HashMap<>(); List policies = getPolicy(currentResources); if (CollectionUtils.isNotEmpty(policies)) { @@ -136,10 +137,12 @@ protected void updatePolicy(List currentResources, Map u updatedPolicyResourcesList.add(resource); } } + updatedAttributes.put(ATTR_POLICY_RESOURCES, updatedPolicyResourcesList); policyVertex.removeProperty(ATTR_POLICY_RESOURCES); policyEntity.setAttribute(ATTR_POLICY_RESOURCES, updatedPolicyResourcesList); context.addUpdated(policyEntity.getGuid(), policyEntity, entityType, policyVertex); + recordUpdatedChildEntities(policyVertex, updatedAttributes); this.preProcessor.processAttributes(policyEntity, context, EntityMutations.EntityOperation.UPDATE); } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index 4efce96a0e5..b7ff63d614a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -115,6 +115,7 @@ private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws String newParentDomainQualifiedName = ""; String superDomainQualifiedName = ""; + // Validate Relationship if(entity.hasRelationshipAttribute(SUB_DOMAIN_REL_TYPE) || entity.hasRelationshipAttribute(DATA_PRODUCT_REL_TYPE)){ throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Cannot update Domain with subDomains or dataProducts"); } From 21d33fd69acfac94f97f058386cd4ab3af726b86 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Mon, 6 May 2024 10:04:38 +0530 Subject: [PATCH 127/316] feat: add policies for feature flag --- addons/policies/bootstrap_admin_policies.json | 25 +++++++++++++++++++ .../service-defs/atlas-servicedef-atlas.json | 5 ++++ .../atlas/authorize/AtlasPrivilege.java | 4 ++- .../atlas/web/resources/AdminResource.java | 6 +++-- 4 files changed, 37 insertions(+), 3 deletions(-) diff --git a/addons/policies/bootstrap_admin_policies.json b/addons/policies/bootstrap_admin_policies.json index 3cd9d7a62fa..5f301ad0af7 100644 --- a/addons/policies/bootstrap_admin_policies.json +++ b/addons/policies/bootstrap_admin_policies.json @@ -49,6 +49,31 @@ "admin-task-cud" ] } + }, + { + "typeName": "AuthPolicy", + "attributes": { + "name": "ADMIN_ALLOW_FEATURE_FLAG_CUD", + "qualifiedName": "ADMIN_ALLOW_FEATURE_FLAG_CUD", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": [ + "service-account-atlan-argo", + "service-account-atlan-backend" + ], + "policyGroups": [], + "policyRoles": [], + "policyResourceCategory": "ADMIN", + "policyResources": [ + "atlas-service:*" + ], + "policyActions": [ + "admin-featureFlag-cud" + ] + } } ] } \ No newline at end of file diff --git a/auth-agents-common/src/main/resources/service-defs/atlas-servicedef-atlas.json b/auth-agents-common/src/main/resources/service-defs/atlas-servicedef-atlas.json index 0539a562b9b..dc8a72c20d4 100644 --- a/auth-agents-common/src/main/resources/service-defs/atlas-servicedef-atlas.json +++ b/auth-agents-common/src/main/resources/service-defs/atlas-servicedef-atlas.json @@ -450,6 +450,11 @@ "itemId": 24, "name": "admin-task-cud", "label": "Admin task CUD API" + }, + { + "itemId": 25, + "name": "admin-featureFlag-cud", + "label": "Admin featureflag CUD API" } ], diff --git a/authorization/src/main/java/org/apache/atlas/authorize/AtlasPrivilege.java b/authorization/src/main/java/org/apache/atlas/authorize/AtlasPrivilege.java index aaf02cbe7a7..29d332cad1d 100644 --- a/authorization/src/main/java/org/apache/atlas/authorize/AtlasPrivilege.java +++ b/authorization/src/main/java/org/apache/atlas/authorize/AtlasPrivilege.java @@ -51,7 +51,9 @@ public enum AtlasPrivilege { ADMIN_ENTITY_AUDITS("admin-entity-audits"), ADMIN_REPAIR_INDEX("admin-repair-index"), - ADMIN_TASK_CUD("admin-task-cud"); + ADMIN_TASK_CUD("admin-task-cud"), + + ADMIN_FEATURE_FLAG_CUD("admin-featureFlag-cud"); private final String type; AtlasPrivilege(String actionType){ diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java index c8715a38680..fddb29b5966 100755 --- a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java +++ b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java @@ -934,14 +934,16 @@ public Map getDebugMetrics() { @POST @Path("featureFlag") @Produces(MediaType.APPLICATION_JSON) - public void setFeatureFlag(@QueryParam("key") String key, @QueryParam("value") String value) { + public void setFeatureFlag(@QueryParam("key") String key, @QueryParam("value") String value) throws AtlasBaseException { + AtlasAuthorizationUtils.verifyAccess(new AtlasAdminAccessRequest(AtlasPrivilege.ADMIN_FEATURE_FLAG_CUD), "featureFlag"); FeatureFlagStore.setFlag(key, value); } @DELETE @Path("featureFlag/{flag}") @Produces(MediaType.APPLICATION_JSON) - public void deleteFeatureFlag(@PathParam("flag") String key) { + public void deleteFeatureFlag(@PathParam("flag") String key) throws AtlasBaseException { + AtlasAuthorizationUtils.verifyAccess(new AtlasAdminAccessRequest(AtlasPrivilege.ADMIN_FEATURE_FLAG_CUD), "featureFlag"); FeatureFlagStore.deleteFlag(key); } private String getEditableEntityTypes(Configuration config) { From e093db2ed46221936a7b06c077535cb1b451efb0 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Mon, 6 May 2024 12:54:56 +0530 Subject: [PATCH 128/316] DG-996 Fix case 4: restrict via relationship API --- .../graph/v2/AtlasRelationshipStoreV2.java | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java index 3e8c8b9e424..5e9d79b28c7 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java @@ -74,6 +74,7 @@ import static org.apache.atlas.repository.Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY; import static org.apache.atlas.repository.Constants.RELATIONSHIP_GUID_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.VERSION_PROPERTY_KEY; +import static org.apache.atlas.repository.graph.GraphHelper.getTypeName; import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.*; import static org.apache.atlas.repository.store.graph.v2.tasks.ClassificationPropagateTaskFactory.CLASSIFICATION_PROPAGATION_RELATIONSHIP_UPDATE; @@ -104,6 +105,11 @@ public class AtlasRelationshipStoreV2 implements AtlasRelationshipStore { private static final String END_2_DOC_ID_KEY = "end2DocId"; private static final String ES_DOC_ID_MAP_KEY = "esDocIdMap"; + private static Set EXCLUDE_MUTATION_REL_TYPE_NAMES = new HashSet() {{ + add("parent_domain_sub_domains"); + add("data_domain_data_products"); + }}; + public enum RelationshipMutation { RELATIONSHIP_CREATE, RELATIONSHIP_UPDATE, @@ -129,6 +135,8 @@ public AtlasRelationship create(AtlasRelationship relationship) throws AtlasBase LOG.debug("==> create({})", relationship); } + validateRelationshipType(relationship.getTypeName()); + AtlasVertex end1Vertex = getVertexFromEndPoint(relationship.getEnd1()); AtlasVertex end2Vertex = getVertexFromEndPoint(relationship.getEnd2()); @@ -161,6 +169,8 @@ public AtlasRelationship update(AtlasRelationship relationship) throws AtlasBase AtlasVertex end1Vertex = edge.getOutVertex(); AtlasVertex end2Vertex = edge.getInVertex(); + validateRelationshipType(edgeType); + // update shouldn't change endType if (StringUtils.isNotEmpty(relationship.getTypeName()) && !StringUtils.equalsIgnoreCase(edgeType, relationship.getTypeName())) { throw new AtlasBaseException(AtlasErrorCode.RELATIONSHIP_UPDATE_TYPE_CHANGE_NOT_ALLOWED, guid, edgeType, relationship.getTypeName()); @@ -320,6 +330,8 @@ public void deleteByIds(List guids) throws AtlasBaseException { throw new AtlasBaseException(AtlasErrorCode.RELATIONSHIP_ALREADY_DELETED, guid); } + validateRelationshipType(getTypeName(edge)); + edgesToDelete.add(edge); AtlasRelationship relationshipToDelete = entityRetriever.mapEdgeToAtlasRelationship(edge); deletedRelationships.add(relationshipToDelete); @@ -368,6 +380,9 @@ public void deleteById(String guid, boolean forceDelete) throws AtlasBaseExcepti if (getState(edge) == DELETED) { throw new AtlasBaseException(AtlasErrorCode.RELATIONSHIP_ALREADY_DELETED, guid); } + + validateRelationshipType(getTypeName(edge)); + deleteDelegate.getHandler().resetHasLineageOnInputOutputDelete(Collections.singleton(edge), null); deleteDelegate.getHandler().deleteRelationships(Collections.singleton(edge), forceDelete); @@ -999,4 +1014,11 @@ private static void setEdgeVertexIdsInContext(AtlasEdge edge) { RequestContext.get().addRelationshipEndToVertexIdMapping(GraphHelper.getAtlasObjectIdForOutVertex(edge), edge.getOutVertex().getId()); RequestContext.get().addRelationshipEndToVertexIdMapping(GraphHelper.getAtlasObjectIdForInVertex(edge), edge.getInVertex().getId()); } + + private static void validateRelationshipType(String relationshipTypeName) throws AtlasBaseException { + if (EXCLUDE_MUTATION_REL_TYPE_NAMES.contains(relationshipTypeName)) { + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, + String.format("Mutating relationship of type %s is not supported via relationship APIs, please use entity APIs", relationshipTypeName)); + } + } } \ No newline at end of file From 2528b6ee05de431635bce6b85e661a8509d59720 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Mon, 6 May 2024 14:59:30 +0530 Subject: [PATCH 129/316] removed featureFlagStore from AuthPolicyPreProcessor --- .../atlas/repository/store/graph/v2/AtlasEntityStoreV2.java | 6 +++--- .../store/graph/v2/preprocessor/AuthPolicyPreProcessor.java | 6 +----- .../preprocessor/datamesh/AbstractDomainPreProcessor.java | 6 ++---- .../v2/preprocessor/datamesh/DataProductPreProcessor.java | 6 ++---- .../graph/v2/preprocessor/datamesh/DomainPreProcessor.java | 5 ++--- 5 files changed, 10 insertions(+), 19 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 2e51f45db59..6689665e55e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -1803,11 +1803,11 @@ public PreProcessor getPreProcessor(String typeName) { break; case DATA_DOMAIN_ENTITY_TYPE: - preProcessor = new DomainPreProcessor(typeRegistry, entityRetriever, graph, featureFlagStore); + preProcessor = new DomainPreProcessor(typeRegistry, entityRetriever, graph); break; case DATA_PRODUCT_ENTITY_TYPE: - preProcessor = new DataProductPreProcessor(typeRegistry, entityRetriever, graph, featureFlagStore); + preProcessor = new DataProductPreProcessor(typeRegistry, entityRetriever, graph); break; case QUERY_ENTITY_TYPE: @@ -1831,7 +1831,7 @@ public PreProcessor getPreProcessor(String typeName) { break; case POLICY_ENTITY_TYPE: - preProcessor = new AuthPolicyPreProcessor(graph, typeRegistry, entityRetriever, featureFlagStore); + preProcessor = new AuthPolicyPreProcessor(graph, typeRegistry, entityRetriever); break; case CONNECTION_ENTITY_TYPE: diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java index e2bb512467b..bf2e25437cf 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java @@ -24,7 +24,6 @@ import org.apache.atlas.authorize.AtlasEntityAccessRequest; import org.apache.atlas.authorize.AtlasPrivilege; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.featureflag.FeatureFlagStore; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; import org.apache.atlas.model.instance.AtlasEntityHeader; @@ -70,17 +69,14 @@ public class AuthPolicyPreProcessor implements PreProcessor { private final AtlasGraph graph; private final AtlasTypeRegistry typeRegistry; private final EntityGraphRetriever entityRetriever; - private final FeatureFlagStore featureFlagStore ; private IndexAliasStore aliasStore; public AuthPolicyPreProcessor(AtlasGraph graph, AtlasTypeRegistry typeRegistry, - EntityGraphRetriever entityRetriever, - FeatureFlagStore featureFlagStore) { + EntityGraphRetriever entityRetriever) { this.graph = graph; this.typeRegistry = typeRegistry; this.entityRetriever = entityRetriever; - this.featureFlagStore = featureFlagStore; aliasStore = new ESAliasStore(graph, entityRetriever); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 8487715813f..036e46dce1a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -60,14 +60,12 @@ public abstract class AbstractDomainPreProcessor implements PreProcessor { protected final AtlasTypeRegistry typeRegistry; protected final EntityGraphRetriever entityRetriever; private final PreProcessor preProcessor; - private final FeatureFlagStore featureFlagStore; protected EntityDiscoveryService discovery; - AbstractDomainPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, AtlasGraph graph, FeatureFlagStore featureFlagStore) { + AbstractDomainPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, AtlasGraph graph) { this.entityRetriever = entityRetriever; this.typeRegistry = typeRegistry; - this.preProcessor = new AuthPolicyPreProcessor(graph, typeRegistry, entityRetriever, featureFlagStore); - this.featureFlagStore = featureFlagStore; + this.preProcessor = new AuthPolicyPreProcessor(graph, typeRegistry, entityRetriever); try { this.discovery = new EntityDiscoveryService(typeRegistry, graph, null, null, null, null); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 02cc33bb19b..d3178dbcefc 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -3,7 +3,6 @@ import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.RequestContext; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.featureflag.FeatureFlagStore; import org.apache.atlas.model.instance.*; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; @@ -12,7 +11,6 @@ import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,8 +28,8 @@ public class DataProductPreProcessor extends AbstractDomainPreProcessor { private List currentResources; private Map updatedPolicyResources; public DataProductPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, - AtlasGraph graph, FeatureFlagStore featureFlagStore) { - super(typeRegistry, entityRetriever, graph, featureFlagStore); + AtlasGraph graph) { + super(typeRegistry, entityRetriever, graph); this.currentResources = new ArrayList<>(); this.updatedPolicyResources = new HashMap<>(); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index b7ff63d614a..1faa9b23133 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -36,7 +36,6 @@ import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -56,8 +55,8 @@ public class DomainPreProcessor extends AbstractDomainPreProcessor { private Map updatedPolicyResources; public DomainPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, - AtlasGraph graph, FeatureFlagStore featureFlagStore) { - super(typeRegistry, entityRetriever, graph, featureFlagStore); + AtlasGraph graph) { + super(typeRegistry, entityRetriever, graph); this.currentResources = new ArrayList<>(); this.updatedPolicyResources = new HashMap<>(); } From 3d814b4df49cf695132312cb148bd87eabd1218a Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Mon, 6 May 2024 15:32:31 +0530 Subject: [PATCH 130/316] Revert "removed featureFlagStore from AuthPolicyPreProcessor" This reverts commit 2528b6ee05de431635bce6b85e661a8509d59720. --- .../atlas/repository/store/graph/v2/AtlasEntityStoreV2.java | 6 +++--- .../store/graph/v2/preprocessor/AuthPolicyPreProcessor.java | 6 +++++- .../preprocessor/datamesh/AbstractDomainPreProcessor.java | 6 ++++-- .../v2/preprocessor/datamesh/DataProductPreProcessor.java | 6 ++++-- .../graph/v2/preprocessor/datamesh/DomainPreProcessor.java | 5 +++-- 5 files changed, 19 insertions(+), 10 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 6689665e55e..2e51f45db59 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -1803,11 +1803,11 @@ public PreProcessor getPreProcessor(String typeName) { break; case DATA_DOMAIN_ENTITY_TYPE: - preProcessor = new DomainPreProcessor(typeRegistry, entityRetriever, graph); + preProcessor = new DomainPreProcessor(typeRegistry, entityRetriever, graph, featureFlagStore); break; case DATA_PRODUCT_ENTITY_TYPE: - preProcessor = new DataProductPreProcessor(typeRegistry, entityRetriever, graph); + preProcessor = new DataProductPreProcessor(typeRegistry, entityRetriever, graph, featureFlagStore); break; case QUERY_ENTITY_TYPE: @@ -1831,7 +1831,7 @@ public PreProcessor getPreProcessor(String typeName) { break; case POLICY_ENTITY_TYPE: - preProcessor = new AuthPolicyPreProcessor(graph, typeRegistry, entityRetriever); + preProcessor = new AuthPolicyPreProcessor(graph, typeRegistry, entityRetriever, featureFlagStore); break; case CONNECTION_ENTITY_TYPE: diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java index bf2e25437cf..e2bb512467b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java @@ -24,6 +24,7 @@ import org.apache.atlas.authorize.AtlasEntityAccessRequest; import org.apache.atlas.authorize.AtlasPrivilege; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.featureflag.FeatureFlagStore; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; import org.apache.atlas.model.instance.AtlasEntityHeader; @@ -69,14 +70,17 @@ public class AuthPolicyPreProcessor implements PreProcessor { private final AtlasGraph graph; private final AtlasTypeRegistry typeRegistry; private final EntityGraphRetriever entityRetriever; + private final FeatureFlagStore featureFlagStore ; private IndexAliasStore aliasStore; public AuthPolicyPreProcessor(AtlasGraph graph, AtlasTypeRegistry typeRegistry, - EntityGraphRetriever entityRetriever) { + EntityGraphRetriever entityRetriever, + FeatureFlagStore featureFlagStore) { this.graph = graph; this.typeRegistry = typeRegistry; this.entityRetriever = entityRetriever; + this.featureFlagStore = featureFlagStore; aliasStore = new ESAliasStore(graph, entityRetriever); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 036e46dce1a..8487715813f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -60,12 +60,14 @@ public abstract class AbstractDomainPreProcessor implements PreProcessor { protected final AtlasTypeRegistry typeRegistry; protected final EntityGraphRetriever entityRetriever; private final PreProcessor preProcessor; + private final FeatureFlagStore featureFlagStore; protected EntityDiscoveryService discovery; - AbstractDomainPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, AtlasGraph graph) { + AbstractDomainPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, AtlasGraph graph, FeatureFlagStore featureFlagStore) { this.entityRetriever = entityRetriever; this.typeRegistry = typeRegistry; - this.preProcessor = new AuthPolicyPreProcessor(graph, typeRegistry, entityRetriever); + this.preProcessor = new AuthPolicyPreProcessor(graph, typeRegistry, entityRetriever, featureFlagStore); + this.featureFlagStore = featureFlagStore; try { this.discovery = new EntityDiscoveryService(typeRegistry, graph, null, null, null, null); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index d3178dbcefc..02cc33bb19b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -3,6 +3,7 @@ import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.RequestContext; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.featureflag.FeatureFlagStore; import org.apache.atlas.model.instance.*; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; @@ -11,6 +12,7 @@ import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -28,8 +30,8 @@ public class DataProductPreProcessor extends AbstractDomainPreProcessor { private List currentResources; private Map updatedPolicyResources; public DataProductPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, - AtlasGraph graph) { - super(typeRegistry, entityRetriever, graph); + AtlasGraph graph, FeatureFlagStore featureFlagStore) { + super(typeRegistry, entityRetriever, graph, featureFlagStore); this.currentResources = new ArrayList<>(); this.updatedPolicyResources = new HashMap<>(); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java index 1faa9b23133..b7ff63d614a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java @@ -36,6 +36,7 @@ import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,8 +56,8 @@ public class DomainPreProcessor extends AbstractDomainPreProcessor { private Map updatedPolicyResources; public DomainPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, - AtlasGraph graph) { - super(typeRegistry, entityRetriever, graph); + AtlasGraph graph, FeatureFlagStore featureFlagStore) { + super(typeRegistry, entityRetriever, graph, featureFlagStore); this.currentResources = new ArrayList<>(); this.updatedPolicyResources = new HashMap<>(); } From 1b84c26ab30e68c9dc4c25f263775143fe635ef4 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Mon, 6 May 2024 17:53:42 +0530 Subject: [PATCH 131/316] feat: add feature for temp index --- .../org/apache/atlas/repository/Constants.java | 15 +++++++++++++++ .../apache/atlas/service}/FeatureFlagStore.java | 2 +- .../atlas/discovery/EntityDiscoveryService.java | 11 ++++++----- .../atlas/repository/util/AccessControlUtils.java | 13 +++---------- .../atlas/web/filters/ActiveServerFilter.java | 4 ++-- .../apache/atlas/web/resources/AdminResource.java | 2 +- 6 files changed, 28 insertions(+), 19 deletions(-) rename {graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus => common/src/main/java/org/apache/atlas/service}/FeatureFlagStore.java (96%) diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index e10796992d3..380be3e4490 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -19,7 +19,9 @@ package org.apache.atlas.repository; import org.apache.atlas.ApplicationProperties; +import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasException; +import org.apache.atlas.service.FeatureFlagStore; import org.apache.commons.configuration.Configuration; import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; @@ -453,6 +455,19 @@ private static String getEncodedTypePropertyKey(String defaultKey) { } } + public static String getESIndex() { + String indexSuffix = null; + if(AtlasConfiguration.ATLAS_MAINTENANCE_MODE.getBoolean()) { + try { + if (FeatureFlagStore.evaluate("use_temp_es_index", "true")) { + indexSuffix = "_temp"; + } + } catch (Exception ignored) { + } + } + return indexSuffix == null ? VERTEX_INDEX_NAME : VERTEX_INDEX_NAME + indexSuffix; + } + public static String getStaticFileAsString(String fileName) throws IOException { String atlasHomeDir = System.getProperty("atlas.home"); atlasHomeDir = StringUtils.isEmpty(atlasHomeDir) ? "." : atlasHomeDir; diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/FeatureFlagStore.java b/common/src/main/java/org/apache/atlas/service/FeatureFlagStore.java similarity index 96% rename from graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/FeatureFlagStore.java rename to common/src/main/java/org/apache/atlas/service/FeatureFlagStore.java index 87b68635efb..adfca599d12 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/FeatureFlagStore.java +++ b/common/src/main/java/org/apache/atlas/service/FeatureFlagStore.java @@ -1,4 +1,4 @@ -package org.apache.atlas.repository.graphdb.janus; +package org.apache.atlas.service; import org.apache.atlas.service.redis.RedisService; import org.apache.commons.lang.StringUtils; diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index b1b120c3fbb..f2dab34a1a5 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -45,6 +45,7 @@ import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.repository.userprofile.UserProfileService; import org.apache.atlas.searchlog.ESSearchLogger; +import org.apache.atlas.service.FeatureFlagStore; import org.apache.atlas.stats.StatsClient; import org.apache.atlas.type.*; import org.apache.atlas.type.AtlasBuiltInTypes.AtlasObjectIdType; @@ -75,9 +76,7 @@ import static org.apache.atlas.SortOrder.ASCENDING; import static org.apache.atlas.model.instance.AtlasEntity.Status.ACTIVE; import static org.apache.atlas.model.instance.AtlasEntity.Status.DELETED; -import static org.apache.atlas.repository.Constants.ASSET_ENTITY_TYPE; -import static org.apache.atlas.repository.Constants.OWNER_ATTRIBUTE; -import static org.apache.atlas.repository.Constants.VERTEX_INDEX_NAME; +import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.BASIC_SEARCH_STATE_FILTER; import static org.apache.atlas.util.AtlasGremlinQueryProvider.AtlasGremlinQuery.TO_RANGE_LIST; @@ -1134,8 +1133,10 @@ public List searchUsingTermQualifiedName(int from, int size, } private String getIndexName(IndexSearchParams params) throws AtlasBaseException { + String vertexIndexName = getESIndex(); + if (StringUtils.isEmpty(params.getPersona()) && StringUtils.isEmpty(params.getPurpose())) { - return VERTEX_INDEX_NAME; + return vertexIndexName; } String qualifiedName = ""; @@ -1151,7 +1152,7 @@ private String getIndexName(IndexSearchParams params) throws AtlasBaseException if (StringUtils.isNotEmpty(aliasName)) { if(params.isAccessControlExclusive()) { accessControlExclusiveDsl(params, aliasName); - aliasName = aliasName+","+VERTEX_INDEX_NAME; + aliasName = aliasName+","+vertexIndexName; } return aliasName; } else { diff --git a/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java b/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java index d4686a9562a..4588bdd84d5 100644 --- a/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java @@ -45,15 +45,7 @@ import static org.apache.atlas.AtlasErrorCode.ACCESS_CONTROL_ALREADY_EXISTS; import static org.apache.atlas.AtlasErrorCode.DISABLED_OPERATION; import static org.apache.atlas.AtlasErrorCode.OPERATION_NOT_SUPPORTED; -import static org.apache.atlas.repository.Constants.ATTR_ADMIN_GROUPS; -import static org.apache.atlas.repository.Constants.ATTR_ADMIN_ROLES; -import static org.apache.atlas.repository.Constants.ATTR_ADMIN_USERS; -import static org.apache.atlas.repository.Constants.ATTR_TENANT_ID; -import static org.apache.atlas.repository.Constants.CONNECTION_ENTITY_TYPE; -import static org.apache.atlas.repository.Constants.DEFAULT_TENANT_ID; -import static org.apache.atlas.repository.Constants.NAME; -import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; -import static org.apache.atlas.repository.Constants.VERTEX_INDEX_NAME; +import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.util.AtlasEntityUtils.getListAttribute; import static org.apache.atlas.repository.util.AtlasEntityUtils.getQualifiedName; import static org.apache.atlas.repository.util.AtlasEntityUtils.getStringAttribute; @@ -379,7 +371,8 @@ public static void validateUniquenessByTags(AtlasGraph graph, List tags, private static boolean hasMatchingVertex(AtlasGraph graph, List newTags, IndexSearchParams indexSearchParams) throws AtlasBaseException { - AtlasIndexQuery indexQuery = graph.elasticsearchQuery(VERTEX_INDEX_NAME); + String vertexIndexName = getESIndex(); + AtlasIndexQuery indexQuery = graph.elasticsearchQuery(vertexIndexName); DirectIndexQueryResult indexQueryResult = indexQuery.vertices(indexSearchParams); Iterator iterator = indexQueryResult.getIterator(); diff --git a/webapp/src/main/java/org/apache/atlas/web/filters/ActiveServerFilter.java b/webapp/src/main/java/org/apache/atlas/web/filters/ActiveServerFilter.java index 1abdd256664..2721efa6f25 100644 --- a/webapp/src/main/java/org/apache/atlas/web/filters/ActiveServerFilter.java +++ b/webapp/src/main/java/org/apache/atlas/web/filters/ActiveServerFilter.java @@ -21,7 +21,7 @@ import org.apache.atlas.AtlasConfiguration; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.repository.graphdb.janus.FeatureFlagStore; +import org.apache.atlas.service.FeatureFlagStore; import org.apache.atlas.type.AtlasType; import org.apache.atlas.web.service.ActiveInstanceState; import org.apache.atlas.web.service.ServiceState; @@ -58,7 +58,7 @@ public class ActiveServerFilter implements Filter { private static final Logger LOG = LoggerFactory.getLogger(ActiveServerFilter.class); private static final String MIGRATION_STATUS_STATIC_PAGE = "migration-status.html"; private static final String[] WHITELISTED_APIS_SIGNATURE = {"search", "lineage", "auditSearch", "accessors" - , "evaluator"}; + , "evaluator", "featureFlag"}; private static final String DISABLE_WRITE_FLAG = "disable_writes"; private final ActiveInstanceState activeInstanceState; diff --git a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java index fddb29b5966..71d773a8791 100755 --- a/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java +++ b/webapp/src/main/java/org/apache/atlas/web/resources/AdminResource.java @@ -48,7 +48,6 @@ import org.apache.atlas.model.patches.AtlasPatch.AtlasPatches; import org.apache.atlas.model.tasks.AtlasTask; import org.apache.atlas.repository.audit.AtlasAuditService; -import org.apache.atlas.repository.graphdb.janus.FeatureFlagStore; import org.apache.atlas.repository.impexp.AtlasServerService; import org.apache.atlas.repository.impexp.ExportImportAuditService; import org.apache.atlas.repository.impexp.ExportService; @@ -57,6 +56,7 @@ import org.apache.atlas.repository.impexp.ZipSink; import org.apache.atlas.repository.patches.AtlasPatchManager; import org.apache.atlas.repository.store.graph.AtlasEntityStore; +import org.apache.atlas.service.FeatureFlagStore; import org.apache.atlas.service.metrics.MetricsRegistry; import org.apache.atlas.services.MetricsService; import org.apache.atlas.tasks.TaskManagement; From 71b757f0645ab5647e911e2047dead73bf5a8889 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Mon, 6 May 2024 17:58:11 +0530 Subject: [PATCH 132/316] feat: keep maintenance mode off --- intg/src/main/java/org/apache/atlas/AtlasConfiguration.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index 02adc39be61..a701510a4e0 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -113,7 +113,7 @@ public enum AtlasConfiguration { INDEXSEARCH_ASYNC_SEARCH_KEEP_ALIVE_TIME_IN_SECONDS("atlas.indexsearch.async.search.keep.alive.time.in.seconds", 300), - ATLAS_MAINTENANCE_MODE("atlas.maintenance.mode", true); + ATLAS_MAINTENANCE_MODE("atlas.maintenance.mode", false); private static final Configuration APPLICATION_PROPERTIES; From 5f894960817fa1d771fd114d9e44aad5023647fa Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Mon, 6 May 2024 19:28:19 +0530 Subject: [PATCH 133/316] fix: add proper error logs incase feature flag evaluation fails --- .../src/main/java/org/apache/atlas/repository/Constants.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 380be3e4490..a4a0d83efe8 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -462,7 +462,8 @@ public static String getESIndex() { if (FeatureFlagStore.evaluate("use_temp_es_index", "true")) { indexSuffix = "_temp"; } - } catch (Exception ignored) { + } catch (Exception e) { + LOG.error("Failed to evaluate feature flag with error", e); } } return indexSuffix == null ? VERTEX_INDEX_NAME : VERTEX_INDEX_NAME + indexSuffix; From 531921a2ba5ce22838236b682fd82e37fa0bca5a Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Mon, 6 May 2024 19:12:31 +0530 Subject: [PATCH 134/316] DG-996 Code refacotrings --- .../apache/atlas/repository/Constants.java | 3 - .../store/graph/v2/AtlasEntityStoreV2.java | 8 +- .../preprocessor/AuthPolicyPreProcessor.java | 5 +- .../v2/preprocessor/PreProcessorUtils.java | 25 +-- .../datamesh/AbstractDomainPreProcessor.java | 89 +++++++--- ...essor.java => DataDomainPreProcessor.java} | 167 +++++++----------- .../datamesh/DataProductPreProcessor.java | 146 ++++++--------- 7 files changed, 194 insertions(+), 249 deletions(-) rename repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/{DomainPreProcessor.java => DataDomainPreProcessor.java} (68%) diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 42ad0d9e26e..f942a068396 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -139,9 +139,6 @@ public final class Constants { */ public static final String DATA_DOMAIN_ENTITY_TYPE = "DataDomain"; public static final String DATA_PRODUCT_ENTITY_TYPE = "DataProduct"; - public static final String DATA_PRODUCT_EDGE_LABEL = "__DataDomain.dataProducts"; - public static final String DOMAIN_PARENT_EDGE_LABEL = "__DataDomain.subDomains"; - /** diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 2e51f45db59..912d1cd5beb 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -63,7 +63,7 @@ import org.apache.atlas.repository.store.graph.v2.preprocessor.accesscontrol.PersonaPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.accesscontrol.PurposePreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.datamesh.DataProductPreProcessor; -import org.apache.atlas.repository.store.graph.v2.preprocessor.datamesh.DomainPreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.datamesh.DataDomainPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.glossary.CategoryPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.glossary.GlossaryPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.glossary.TermPreProcessor; @@ -1803,11 +1803,11 @@ public PreProcessor getPreProcessor(String typeName) { break; case DATA_DOMAIN_ENTITY_TYPE: - preProcessor = new DomainPreProcessor(typeRegistry, entityRetriever, graph, featureFlagStore); + preProcessor = new DataDomainPreProcessor(typeRegistry, entityRetriever, graph); break; case DATA_PRODUCT_ENTITY_TYPE: - preProcessor = new DataProductPreProcessor(typeRegistry, entityRetriever, graph, featureFlagStore); + preProcessor = new DataProductPreProcessor(typeRegistry, entityRetriever, graph); break; case QUERY_ENTITY_TYPE: @@ -1831,7 +1831,7 @@ public PreProcessor getPreProcessor(String typeName) { break; case POLICY_ENTITY_TYPE: - preProcessor = new AuthPolicyPreProcessor(graph, typeRegistry, entityRetriever, featureFlagStore); + preProcessor = new AuthPolicyPreProcessor(graph, typeRegistry, entityRetriever); break; case CONNECTION_ENTITY_TYPE: diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java index e2bb512467b..ce16f0e9ae7 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java @@ -70,17 +70,14 @@ public class AuthPolicyPreProcessor implements PreProcessor { private final AtlasGraph graph; private final AtlasTypeRegistry typeRegistry; private final EntityGraphRetriever entityRetriever; - private final FeatureFlagStore featureFlagStore ; private IndexAliasStore aliasStore; public AuthPolicyPreProcessor(AtlasGraph graph, AtlasTypeRegistry typeRegistry, - EntityGraphRetriever entityRetriever, - FeatureFlagStore featureFlagStore) { + EntityGraphRetriever entityRetriever) { this.graph = graph; this.typeRegistry = typeRegistry; this.entityRetriever = entityRetriever; - this.featureFlagStore = featureFlagStore; aliasStore = new ESAliasStore(graph, entityRetriever); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index 14dcff2665a..c8728df251c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -24,8 +24,6 @@ import static org.apache.atlas.repository.Constants.QUERY_COLLECTION_ENTITY_TYPE; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY; -import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_CATEGORY; -import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_RESOURCES; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; public class PreProcessorUtils { @@ -42,15 +40,18 @@ public class PreProcessorUtils { public static final String GLOSSARY_CATEGORY_REL_TYPE = "AtlasGlossaryCategoryAnchor"; //DataMesh models constants - public static final String DATA_PRODUCT_TYPE = "DataProduct"; - public static final String PARENT_DOMAIN = "parentDomain"; + public static final String PARENT_DOMAIN_REL_TYPE = "parentDomain"; public static final String SUB_DOMAIN_REL_TYPE = "subDomains"; public static final String DATA_PRODUCT_REL_TYPE = "dataProducts"; - public static final String PARENT_DOMAIN_QN = "parentDomainQualifiedName"; - public static final String SUPER_DOMAIN_QN = "superDomainQualifiedName"; - public static final String DATA_DOMAIN = "dataDomain"; + public static final String DATA_DOMAIN_REL_TYPE = "dataDomain"; + public static final String MESH_POLICY_CATEGORY = "datamesh"; - public static final String MIGRATION_CUSTOM_ATTRIBUTE = "isQualifiedNameMigrated"; + + public static final String DATA_PRODUCT_EDGE_LABEL = "__DataDomain.dataProducts"; + public static final String DOMAIN_PARENT_EDGE_LABEL = "__DataDomain.subDomains"; + + public static final String PARENT_DOMAIN_QN_ATTR = "parentDomainQualifiedName"; + public static final String SUPER_DOMAIN_QN_ATTR = "superDomainQualifiedName"; //Query models constants public static final String PREFIX_QUERY_QN = "default/collection/"; @@ -132,6 +133,10 @@ public static List indexSearchPaginated(Map d IndexSearchParams searchParams = new IndexSearchParams(); List ret = new ArrayList<>(); + if (CollectionUtils.isNotEmpty(attributes)) { + searchParams.setAttributes(attributes); + } + List sortList = new ArrayList<>(0); sortList.add(mapOf("__timestamp", mapOf("order", "asc"))); sortList.add(mapOf("__guid", mapOf("order", "asc"))); @@ -145,10 +150,6 @@ public static List indexSearchPaginated(Map d dsl.put("size", size); searchParams.setDsl(dsl); - if (CollectionUtils.isNotEmpty(attributes)) { - searchParams.setAttributes(attributes); - } - List headers = discovery.directIndexSearch(searchParams).getEntities(); if (CollectionUtils.isNotEmpty(headers)) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 8487715813f..4b491683fea 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -17,6 +17,7 @@ */ package org.apache.atlas.repository.store.graph.v2.preprocessor.datamesh; +import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.AtlasException; import org.apache.atlas.RequestContext; import org.apache.atlas.authorize.AtlasAuthorizationUtils; @@ -24,7 +25,6 @@ import org.apache.atlas.authorize.AtlasPrivilege; import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.featureflag.FeatureFlagStore; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasObjectId; @@ -40,7 +40,6 @@ import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -49,8 +48,8 @@ import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.*; -import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_CATEGORY; import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_RESOURCES; +import static org.apache.atlas.repository.util.AccessControlUtils.REL_ATTR_ACCESS_CONTROL; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; public abstract class AbstractDomainPreProcessor implements PreProcessor { @@ -60,14 +59,16 @@ public abstract class AbstractDomainPreProcessor implements PreProcessor { protected final AtlasTypeRegistry typeRegistry; protected final EntityGraphRetriever entityRetriever; private final PreProcessor preProcessor; - private final FeatureFlagStore featureFlagStore; protected EntityDiscoveryService discovery; - AbstractDomainPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, AtlasGraph graph, FeatureFlagStore featureFlagStore) { + private static final Set POLICY_ATTRIBUTES_FOR_SEARCH = new HashSet<>(Arrays.asList(ATTR_POLICY_RESOURCES)); + + static final Set PARENT_ATTRIBUTES = new HashSet<>(Arrays.asList(SUPER_DOMAIN_QN_ATTR, PARENT_DOMAIN_QN_ATTR)); + + AbstractDomainPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, AtlasGraph graph) { this.entityRetriever = entityRetriever; this.typeRegistry = typeRegistry; - this.preProcessor = new AuthPolicyPreProcessor(graph, typeRegistry, entityRetriever, featureFlagStore); - this.featureFlagStore = featureFlagStore; + this.preProcessor = new AuthPolicyPreProcessor(graph, typeRegistry, entityRetriever); try { this.discovery = new EntityDiscoveryService(typeRegistry, graph, null, null, null, null); @@ -103,26 +104,22 @@ protected void isAuthorized(AtlasEntityHeader sourceDomain, AtlasEntityHeader ta } } - protected void updatePolicy(List currentResources, Map updatedPolicyResources, EntityMutationContext context) throws AtlasBaseException { - AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("updateDomainPolicy"); + protected void updatePolicies(Map updatedPolicyResources, EntityMutationContext context) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("updatePolicies"); try { - LOG.info("Updating policies for entities {}", currentResources); - Map updatedAttributes = new HashMap<>(); + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(POLICY_ENTITY_TYPE); - List policies = getPolicy(currentResources); - if (CollectionUtils.isNotEmpty(policies)) { - AtlasEntityType entityType = typeRegistry.getEntityTypeByName(POLICY_ENTITY_TYPE); + List policies = getPolicies(updatedPolicyResources.keySet()); + if (CollectionUtils.isNotEmpty(policies)) { for (AtlasEntityHeader policy : policies) { - LOG.info("Updating policy {}", policy); AtlasVertex policyVertex = entityRetriever.getEntityVertex(policy.getGuid()); AtlasEntity policyEntity = entityRetriever.toAtlasEntity(policyVertex); - String policyCategory = (String) policyEntity.getAttribute(ATTR_POLICY_CATEGORY); - if (policyEntity.hasRelationshipAttribute("accessControl") && !StringUtils.equals(policyCategory, MESH_POLICY_CATEGORY)) { - LOG.info("PolicyCategory {}", policyCategory); - AtlasVertex accessControl = entityRetriever.getEntityVertex(((AtlasObjectId) policyEntity.getRelationshipAttribute("accessControl")).getGuid()); + if (policyEntity.hasRelationshipAttribute(REL_ATTR_ACCESS_CONTROL) && policyEntity.getRelationshipAttribute(REL_ATTR_ACCESS_CONTROL) != null) { + AtlasObjectId accessControlObjId = (AtlasObjectId) policyEntity.getRelationshipAttribute(REL_ATTR_ACCESS_CONTROL); + AtlasVertex accessControl = entityRetriever.getEntityVertex(accessControlObjId.getGuid()); context.getDiscoveryContext().addResolvedGuid(GraphHelper.getGuid(accessControl), accessControl); } @@ -137,24 +134,65 @@ protected void updatePolicy(List currentResources, Map u updatedPolicyResourcesList.add(resource); } } + Map updatedAttributes = new HashMap<>(); updatedAttributes.put(ATTR_POLICY_RESOURCES, updatedPolicyResourcesList); - policyVertex.removeProperty(ATTR_POLICY_RESOURCES); + //policyVertex.removeProperty(ATTR_POLICY_RESOURCES); policyEntity.setAttribute(ATTR_POLICY_RESOURCES, updatedPolicyResourcesList); + context.addUpdated(policyEntity.getGuid(), policyEntity, entityType, policyVertex); recordUpdatedChildEntities(policyVertex, updatedAttributes); this.preProcessor.processAttributes(policyEntity, context, EntityMutations.EntityOperation.UPDATE); } } - }finally { + } finally { RequestContext.get().endMetricRecord(metricRecorder); } + } + + protected void exists(String assetType, String assetName, String parentDomainQualifiedName) throws AtlasBaseException { + boolean exists = false; + + List> mustClauseList = new ArrayList(); + mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", assetType))); + mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); + mustClauseList.add(mapOf("term", mapOf("name.keyword", assetName))); + + + Map bool = new HashMap<>(); + if (StringUtils.isNotEmpty(parentDomainQualifiedName)) { + mustClauseList.add(mapOf("term", mapOf("parentDomainQualifiedName", parentDomainQualifiedName))); + } else { + List> mustNotClauseList = new ArrayList(); + mustNotClauseList.add(mapOf("exists", mapOf("field", "parentDomainQualifiedName"))); + bool.put("must_not", mustNotClauseList); + } + + bool.put("must", mustClauseList); + Map dsl = mapOf("query", mapOf("bool", bool)); + + List assets = indexSearchPaginated(dsl, null, this.discovery); + + if (CollectionUtils.isNotEmpty(assets)) { + for (AtlasEntityHeader asset : assets) { + String name = (String) asset.getAttribute(NAME); + if (assetName.equals(name)) { + exists = true; + break; + } + } + } + + if (exists) { + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, + String.format("%s with name %s already exists in the domain", assetType, assetName)); + } } - protected List getPolicy(List resources) throws AtlasBaseException { - AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("getPolicy"); + protected List getPolicies(Set resources) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("getPolicies"); try { List> mustClauseList = new ArrayList<>(); mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", POLICY_ENTITY_TYPE))); @@ -165,11 +203,8 @@ protected List getPolicy(List resources) throws Atlas bool.put("must", mustClauseList); Map dsl = mapOf("query", mapOf("bool", bool)); - Set attributes = new HashSet<>(Arrays.asList(ATTR_POLICY_RESOURCES, ATTR_POLICY_CATEGORY)); - - List policies = indexSearchPaginated(dsl, attributes, discovery); - return policies; + return indexSearchPaginated(dsl, POLICY_ATTRIBUTES_FOR_SEARCH, discovery); } finally { RequestContext.get().endMetricRecord(metricRecorder); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java similarity index 68% rename from repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java rename to repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java index b7ff63d614a..b30f6b3c3a6 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java @@ -21,7 +21,6 @@ import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.RequestContext; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.featureflag.FeatureFlagStore; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasObjectId; @@ -36,7 +35,6 @@ import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -48,50 +46,47 @@ import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.*; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; -public class DomainPreProcessor extends AbstractDomainPreProcessor { - private static final Logger LOG = LoggerFactory.getLogger(DomainPreProcessor.class); - private AtlasEntityHeader parentDomain; +public class DataDomainPreProcessor extends AbstractDomainPreProcessor { + private static final Logger LOG = LoggerFactory.getLogger(DataDomainPreProcessor.class); + private EntityMutationContext context; - private List currentResources ; private Map updatedPolicyResources; - public DomainPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, - AtlasGraph graph, FeatureFlagStore featureFlagStore) { - super(typeRegistry, entityRetriever, graph, featureFlagStore); - this.currentResources = new ArrayList<>(); + public DataDomainPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, + AtlasGraph graph) { + super(typeRegistry, entityRetriever, graph); this.updatedPolicyResources = new HashMap<>(); } @Override public void processAttributes(AtlasStruct entityStruct, EntityMutationContext context, EntityMutations.EntityOperation operation) throws AtlasBaseException { - //Handle name & qualifiedName if (LOG.isDebugEnabled()) { - LOG.debug("DomainPreProcessor.processAttributes: pre processing {}, {}", + LOG.debug("DataDomainPreProcessor.processAttributes: pre processing {}, {}", entityStruct.getAttribute(QUALIFIED_NAME), operation); } this.context = context; AtlasEntity entity = (AtlasEntity) entityStruct; - AtlasVertex vertex = context.getVertex(entity.getGuid()); - - setParent(entity); switch (operation) { case CREATE: - processCreateDomain(entity, vertex); + processCreateDomain(entity); break; case UPDATE: + AtlasVertex vertex = context.getVertex(entity.getGuid()); processUpdateDomain(entity, vertex); break; } } - private void processCreateDomain(AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { + private void processCreateDomain(AtlasEntity entity) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreateDomain"); String domainName = (String) entity.getAttribute(NAME); - String parentDomainQualifiedName = (String) entity.getAttribute(PARENT_DOMAIN_QN); + String parentDomainQualifiedName = (String) entity.getAttribute(PARENT_DOMAIN_QN_ATTR); + + AtlasEntityHeader parentDomain = getParent(entity); if(parentDomain != null ){ parentDomainQualifiedName = (String) parentDomain.getAttribute(QUALIFIED_NAME); } @@ -103,52 +98,55 @@ private void processCreateDomain(AtlasEntity entity, AtlasVertex vertex) throws private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdateDomain"); - String domainName = (String) entity.getAttribute(NAME); - String vertexQnName = vertex.getProperty(QUALIFIED_NAME, String.class); - AtlasEntityHeader currentParentDomainHeader = null; - String currentParentDomainQualifiedName = ""; + // Validate Relationship + if(entity.hasRelationshipAttribute(SUB_DOMAIN_REL_TYPE) || entity.hasRelationshipAttribute(DATA_PRODUCT_REL_TYPE)){ + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Cannot update Domain's subDomains or dataProducts relations"); + } + + String vertexQnName = vertex.getProperty(QUALIFIED_NAME, String.class); AtlasEntity storedDomain = entityRetriever.toAtlasEntity(vertex); - AtlasRelatedObjectId currentParentDomain = (AtlasRelatedObjectId) storedDomain.getRelationshipAttribute(PARENT_DOMAIN); + AtlasRelatedObjectId currentParentDomainObjectId = (AtlasRelatedObjectId) storedDomain.getRelationshipAttribute(PARENT_DOMAIN_REL_TYPE); + String newSuperDomainQualifiedName = ""; String newParentDomainQualifiedName = ""; - String superDomainQualifiedName = ""; + String currentParentDomainQualifiedName = ""; - // Validate Relationship - if(entity.hasRelationshipAttribute(SUB_DOMAIN_REL_TYPE) || entity.hasRelationshipAttribute(DATA_PRODUCT_REL_TYPE)){ - throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Cannot update Domain with subDomains or dataProducts"); - } + AtlasEntityHeader currentParentDomainHeader = null; - if(currentParentDomain != null){ - currentParentDomainHeader = entityRetriever.toAtlasEntityHeader(currentParentDomain.getGuid()); + if(currentParentDomainObjectId != null){ + currentParentDomainHeader = entityRetriever.toAtlasEntityHeader(currentParentDomainObjectId.getGuid()); currentParentDomainQualifiedName = (String) currentParentDomainHeader.getAttribute(QUALIFIED_NAME); } - if (parentDomain != null) { - newParentDomainQualifiedName = (String) parentDomain.getAttribute(QUALIFIED_NAME); - superDomainQualifiedName = (String) parentDomain.getAttribute(SUPER_DOMAIN_QN); - if(superDomainQualifiedName == null){ - superDomainQualifiedName = newParentDomainQualifiedName; + AtlasEntityHeader newParentDomainHeader = getParent(entity); + if (newParentDomainHeader != null) { + newParentDomainQualifiedName = (String) newParentDomainHeader.getAttribute(QUALIFIED_NAME); + + newSuperDomainQualifiedName = (String) newParentDomainHeader.getAttribute(SUPER_DOMAIN_QN_ATTR); + if(StringUtils.isEmpty(newSuperDomainQualifiedName)) { + newSuperDomainQualifiedName = newParentDomainQualifiedName; } } - if (!currentParentDomainQualifiedName.equals(newParentDomainQualifiedName) && entity.hasRelationshipAttribute(PARENT_DOMAIN)) { - if(storedDomain.getRelationshipAttribute(PARENT_DOMAIN) == null){ - throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Cannot move Root Domain"); + if (!newParentDomainQualifiedName.equals(currentParentDomainQualifiedName) && entity.hasRelationshipAttribute(PARENT_DOMAIN_REL_TYPE)) { + if(storedDomain.getRelationshipAttribute(PARENT_DOMAIN_REL_TYPE) == null){ + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Cannot move Super Domain inside another domain"); } //Auth check - isAuthorized(currentParentDomainHeader, parentDomain); + isAuthorized(currentParentDomainHeader, newParentDomainHeader); - processMoveSubDomainToAnotherDomain(entity, vertex, currentParentDomainQualifiedName, newParentDomainQualifiedName, vertexQnName, superDomainQualifiedName); + processMoveSubDomainToAnotherDomain(entity, vertex, currentParentDomainQualifiedName, newParentDomainQualifiedName, vertexQnName, newSuperDomainQualifiedName); } else { - String vertexName = vertex.getProperty(NAME, String.class); - if (!vertexName.equals(domainName)) { - domainExists(domainName, newParentDomainQualifiedName); - } + String domainCurrentName = vertex.getProperty(NAME, String.class); + String domainNewName = (String) entity.getAttribute(NAME); + if (!domainCurrentName.equals(domainNewName)) { + domainExists(domainNewName, currentParentDomainQualifiedName); + } entity.setAttribute(QUALIFIED_NAME, vertexQnName); } @@ -159,9 +157,9 @@ private void processMoveSubDomainToAnotherDomain(AtlasEntity domain, AtlasVertex domainVertex, String sourceDomainQualifiedName, String targetDomainQualifiedName, - String currentSubDomainQualifiedName, + String currentDomainQualifiedName, String superDomainQualifiedName) throws AtlasBaseException { - AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("processMoveSubDomainToAnotherGlossary"); + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("processMoveSubDomainToAnotherDomain"); try { String domainName = (String) domain.getAttribute(NAME); @@ -171,26 +169,24 @@ private void processMoveSubDomainToAnotherDomain(AtlasEntity domain, domainExists(domainName, targetDomainQualifiedName); - - // Move Sub-Domain as root Domain if(targetDomainQualifiedName.isEmpty()){ - LOG.info("Moving subDomain {} as root Domain", domainName); + //Moving subDomain to make it Super Domain targetDomainQualifiedName = "default"; - updatedQualifiedName = currentSubDomainQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); + updatedQualifiedName = currentDomainQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); domain.setAttribute(QUALIFIED_NAME, updatedQualifiedName); - domain.setAttribute(PARENT_DOMAIN_QN, null); - domain.setAttribute(SUPER_DOMAIN_QN, null); + domain.setAttribute(PARENT_DOMAIN_QN_ATTR, null); + domain.setAttribute(SUPER_DOMAIN_QN_ATTR, null); superDomainQualifiedName = updatedQualifiedName ; } else{ - updatedQualifiedName = currentSubDomainQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); + updatedQualifiedName = currentDomainQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); domain.setAttribute(QUALIFIED_NAME, updatedQualifiedName); - domain.setAttribute(PARENT_DOMAIN_QN, targetDomainQualifiedName); - domain.setAttribute(SUPER_DOMAIN_QN, superDomainQualifiedName); + domain.setAttribute(PARENT_DOMAIN_QN_ATTR, targetDomainQualifiedName); + domain.setAttribute(SUPER_DOMAIN_QN_ATTR, superDomainQualifiedName); } moveChildrenToAnotherDomain(domainVertex, superDomainQualifiedName, null, sourceDomainQualifiedName, targetDomainQualifiedName); - updatePolicy(this.currentResources, this.updatedPolicyResources, this.context); + updatePolicies(this.updatedPolicyResources, this.context); LOG.info("Moved subDomain {} to Domain {}", domainName, targetDomainQualifiedName); @@ -219,14 +215,13 @@ private void moveChildrenToAnotherDomain(AtlasVertex childDomainVertex, updatedAttributes.put(QUALIFIED_NAME, updatedDomainQualifiedName); //change superDomainQN, parentDomainQN - childDomainVertex.setProperty(SUPER_DOMAIN_QN, superDomainQualifiedName); - childDomainVertex.setProperty(PARENT_DOMAIN_QN, parentDomainQualifiedName); + childDomainVertex.setProperty(SUPER_DOMAIN_QN_ATTR, superDomainQualifiedName); + childDomainVertex.setProperty(PARENT_DOMAIN_QN_ATTR, parentDomainQualifiedName); //Store domainPolicies and resources to be updated String currentResource = "entity:"+ currentDomainQualifiedName; String updatedResource = "entity:"+ updatedDomainQualifiedName; this.updatedPolicyResources.put(currentResource, updatedResource); - this.currentResources.add(currentResource); //update system properties GraphHelper.setModifiedByAsString(childDomainVertex, RequestContext.get().getUser()); @@ -274,14 +269,13 @@ private void moveChildDataProductToAnotherDomain(AtlasVertex productVertex, productVertex.setProperty(QUALIFIED_NAME, updatedQualifiedName); updatedAttributes.put(QUALIFIED_NAME, updatedQualifiedName); - productVertex.setProperty(PARENT_DOMAIN_QN, parentDomainQualifiedName); - productVertex.setProperty(SUPER_DOMAIN_QN, superDomainQualifiedName); + productVertex.setProperty(PARENT_DOMAIN_QN_ATTR, parentDomainQualifiedName); + productVertex.setProperty(SUPER_DOMAIN_QN_ATTR, superDomainQualifiedName); //Store domainPolicies and resources to be updated String currentResource = "entity:"+ currentQualifiedName; String updatedResource = "entity:"+ updatedQualifiedName; this.updatedPolicyResources.put(currentResource, updatedResource); - this.currentResources.add(currentResource); //update system properties GraphHelper.setModifiedByAsString(productVertex, RequestContext.get().getUser()); @@ -295,58 +289,23 @@ private void moveChildDataProductToAnotherDomain(AtlasVertex productVertex, } } - private void setParent(AtlasEntity entity) throws AtlasBaseException { - AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("DomainPreProcessor.setParent"); - if (parentDomain == null) { - AtlasObjectId objectId = (AtlasObjectId) entity.getRelationshipAttribute(PARENT_DOMAIN); - Set attributes = new HashSet<>(Arrays.asList(QUALIFIED_NAME, SUPER_DOMAIN_QN, PARENT_DOMAIN_QN, "__typeName")); - parentDomain = getParent(objectId, attributes); - } + private AtlasEntityHeader getParent(AtlasEntity domainEntity) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("DataDomainPreProcessor.getParent"); + + AtlasObjectId objectId = (AtlasObjectId) domainEntity.getRelationshipAttribute(PARENT_DOMAIN_REL_TYPE); + RequestContext.get().endMetricRecord(metricRecorder); + return getParent(objectId, PARENT_ATTRIBUTES); } private void domainExists(String domainName, String parentDomainQualifiedName) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("domainExists"); - - boolean exists = false; try { - List> mustClauseList = new ArrayList(); - mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", DATA_DOMAIN_ENTITY_TYPE))); - mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); - mustClauseList.add(mapOf("term", mapOf("name.keyword", domainName))); - - - Map bool = new HashMap<>(); - if (parentDomain != null && StringUtils.isNotEmpty(parentDomainQualifiedName)) { - mustClauseList.add(mapOf("term", mapOf("parentDomainQualifiedName", parentDomainQualifiedName))); - } else { - List> mustNotClauseList = new ArrayList(); - mustNotClauseList.add(mapOf("exists", mapOf("field", "parentDomainQualifiedName"))); - bool.put("must_not", mustNotClauseList); - } - - bool.put("must", mustClauseList); - - Map dsl = mapOf("query", mapOf("bool", bool)); + exists(DATA_DOMAIN_ENTITY_TYPE, domainName, parentDomainQualifiedName); - List domains = indexSearchPaginated(dsl, null, this.discovery); - - if (CollectionUtils.isNotEmpty(domains)) { - for (AtlasEntityHeader domain : domains) { - String name = (String) domain.getAttribute(NAME); - if (domainName.equals(name)) { - exists = true; - break; - } - } - } } finally { RequestContext.get().endMetricRecord(metricRecorder); } - - if (exists) { - throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, domainName+" already exists"); - } } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 02cc33bb19b..59048a498f2 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -3,7 +3,6 @@ import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.RequestContext; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.featureflag.FeatureFlagStore; import org.apache.atlas.model.instance.*; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; @@ -12,7 +11,6 @@ import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -25,21 +23,19 @@ public class DataProductPreProcessor extends AbstractDomainPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(DataProductPreProcessor.class); - private AtlasEntityHeader parentDomain; + private EntityMutationContext context; - private List currentResources; private Map updatedPolicyResources; + public DataProductPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, - AtlasGraph graph, FeatureFlagStore featureFlagStore) { - super(typeRegistry, entityRetriever, graph, featureFlagStore); - this.currentResources = new ArrayList<>(); + AtlasGraph graph) { + super(typeRegistry, entityRetriever, graph); this.updatedPolicyResources = new HashMap<>(); } @Override public void processAttributes(AtlasStruct entityStruct, EntityMutationContext context, EntityMutations.EntityOperation operation) throws AtlasBaseException { - //Handle name & qualifiedName if (LOG.isDebugEnabled()) { LOG.debug("DataProductPreProcessor.processAttributes: pre processing {}, {}", entityStruct.getAttribute(QUALIFIED_NAME), operation); @@ -47,24 +43,24 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co this.context = context; AtlasEntity entity = (AtlasEntity) entityStruct; - AtlasVertex vertex = context.getVertex(entity.getGuid()); - - setParent(entity); switch (operation) { case CREATE: - processCreateProduct(entity, vertex); + processCreateProduct(entity); break; case UPDATE: - processUpdateDomain(entity, vertex); + AtlasVertex vertex = context.getVertex(entity.getGuid()); + processUpdateProduct(entity, vertex); break; } } - private void processCreateProduct(AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { + private void processCreateProduct(AtlasEntity entity) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreateProduct"); String productName = (String) entity.getAttribute(NAME); - String parentDomainQualifiedName = (String) entity.getAttribute(PARENT_DOMAIN_QN); + String parentDomainQualifiedName = (String) entity.getAttribute(PARENT_DOMAIN_QN_ATTR); + + AtlasEntityHeader parentDomain = getParent(entity); if(parentDomain != null ){ parentDomainQualifiedName = (String) parentDomain.getAttribute(QUALIFIED_NAME); } @@ -74,49 +70,51 @@ private void processCreateProduct(AtlasEntity entity, AtlasVertex vertex) throws RequestContext.get().endMetricRecord(metricRecorder); } - private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { - AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdateDomain"); - String productName = (String) entity.getAttribute(NAME); - String vertexQnName = vertex.getProperty(QUALIFIED_NAME, String.class); + private void processUpdateProduct(AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdateProduct"); - AtlasEntityHeader currentParentDomainHeader = null; - String currentParentDomainQualifiedName = ""; + if(entity.hasRelationshipAttribute(DATA_DOMAIN_REL_TYPE) && entity.getRelationshipAttribute(DATA_DOMAIN_REL_TYPE) == null){ + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "DataProduct can only be moved to another Domain."); + } + + String vertexQnName = vertex.getProperty(QUALIFIED_NAME, String.class); AtlasEntity storedProduct = entityRetriever.toAtlasEntity(vertex); - AtlasRelatedObjectId currentParentDomain = (AtlasRelatedObjectId) storedProduct.getRelationshipAttribute(DATA_DOMAIN); + AtlasRelatedObjectId currentParentDomainObjectId = (AtlasRelatedObjectId) storedProduct.getRelationshipAttribute(DATA_DOMAIN_REL_TYPE); - String newParentDomainQualifiedName = ""; - String superDomainQualifiedName = ""; + String newParentDomainQualifiedName = null; + String currentParentDomainQualifiedName = null; + AtlasEntityHeader currentParentDomainHeader = null; - if(currentParentDomain != null){ - currentParentDomainHeader = entityRetriever.toAtlasEntityHeader(currentParentDomain.getGuid()); + if(currentParentDomainObjectId != null) { + currentParentDomainHeader = entityRetriever.toAtlasEntityHeader(currentParentDomainObjectId.getGuid()); currentParentDomainQualifiedName = (String) currentParentDomainHeader.getAttribute(QUALIFIED_NAME); } - if (parentDomain != null) { - newParentDomainQualifiedName = (String) parentDomain.getAttribute(QUALIFIED_NAME); - superDomainQualifiedName = (String) parentDomain.getAttribute(SUPER_DOMAIN_QN); - if(superDomainQualifiedName == null){ - superDomainQualifiedName = newParentDomainQualifiedName; - } + AtlasEntityHeader newParentDomainHeader = getParent(entity); + if (newParentDomainHeader != null) { + newParentDomainQualifiedName = (String) newParentDomainHeader.getAttribute(QUALIFIED_NAME); } - if(entity.hasRelationshipAttribute(DATA_DOMAIN) && entity.getRelationshipAttribute(DATA_DOMAIN) == null){ - throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "DataProduct can only be moved to another Domain."); - } - - if (!currentParentDomainQualifiedName.equals(newParentDomainQualifiedName) && entity.hasRelationshipAttribute(DATA_DOMAIN)) { + if (newParentDomainQualifiedName != null && !newParentDomainQualifiedName.equals(currentParentDomainQualifiedName)) { //Auth check - isAuthorized(currentParentDomainHeader, parentDomain); + isAuthorized(currentParentDomainHeader, newParentDomainHeader); + + String newSuperDomainQualifiedName = (String) newParentDomainHeader.getAttribute(SUPER_DOMAIN_QN_ATTR); + if(StringUtils.isEmpty(newSuperDomainQualifiedName)){ + newSuperDomainQualifiedName = newParentDomainQualifiedName; + } + + processMoveDataProductToAnotherDomain(entity, currentParentDomainQualifiedName, newParentDomainQualifiedName, vertexQnName, newSuperDomainQualifiedName); - processMoveDataProductToAnotherDomain(entity, currentParentDomainQualifiedName, newParentDomainQualifiedName, vertexQnName, superDomainQualifiedName); - updatePolicy(this.currentResources, this.updatedPolicyResources, this.context); - entity.setAttribute(PARENT_DOMAIN_QN, newParentDomainQualifiedName); + updatePolicies(this.updatedPolicyResources, this.context); } else { - String vertexName = vertex.getProperty(NAME, String.class); - if (!vertexName.equals(productName)) { - productExists(productName, newParentDomainQualifiedName); + String productCurrentName = vertex.getProperty(NAME, String.class); + String productNewName = (String) entity.getAttribute(NAME); + + if (!productCurrentName.equals(productNewName)) { + productExists(productNewName, currentParentDomainQualifiedName); } entity.setAttribute(QUALIFIED_NAME, vertexQnName); } @@ -133,32 +131,26 @@ private void processMoveDataProductToAnotherDomain(AtlasEntity product, try { String productName = (String) product.getAttribute(NAME); - String updatedQualifiedName; - - if(StringUtils.isEmpty(targetDomainQualifiedName)){ - throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "DataProduct can only be moved to another Domain."); - } LOG.info("Moving dataProduct {} to Domain {}", productName, targetDomainQualifiedName); productExists(productName, targetDomainQualifiedName); + String updatedQualifiedName; if(StringUtils.isEmpty(sourceDomainQualifiedName)){ updatedQualifiedName = targetDomainQualifiedName + "/" + product.getAttribute(QUALIFIED_NAME); - } - else{ + } else { updatedQualifiedName = currentDataProductQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); } product.setAttribute(QUALIFIED_NAME, updatedQualifiedName); - product.setAttribute(PARENT_DOMAIN_QN, targetDomainQualifiedName); - product.setAttribute(SUPER_DOMAIN_QN, superDomainQualifiedName); + product.setAttribute(PARENT_DOMAIN_QN_ATTR, targetDomainQualifiedName); + product.setAttribute(SUPER_DOMAIN_QN_ATTR, superDomainQualifiedName); //Store domainPolicies and resources to be updated String currentResource = "entity:"+ currentDataProductQualifiedName; String updatedResource = "entity:"+ updatedQualifiedName; this.updatedPolicyResources.put(currentResource, updatedResource); - this.currentResources.add(currentResource); LOG.info("Moved dataProduct {} to Domain {}", productName, targetDomainQualifiedName); @@ -167,60 +159,24 @@ private void processMoveDataProductToAnotherDomain(AtlasEntity product, } } - private void setParent(AtlasEntity entity) throws AtlasBaseException { - AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("DataProductPreProcessor.setParent"); - if (parentDomain == null) { - Object relationshipAttribute = entity.getRelationshipAttribute(DATA_DOMAIN); - Set attributes = new HashSet<>(Arrays.asList(QUALIFIED_NAME, SUPER_DOMAIN_QN, PARENT_DOMAIN_QN, "__typeName")); + private AtlasEntityHeader getParent(AtlasEntity productEntity) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("DataProductPreProcessor.getParent"); - parentDomain = getParent(relationshipAttribute, attributes); + Object relationshipAttribute = productEntity.getRelationshipAttribute(DATA_DOMAIN_REL_TYPE); - } RequestContext.get().endMetricRecord(metricRecorder); + return getParent(relationshipAttribute, PARENT_ATTRIBUTES); } private void productExists(String productName, String parentDomainQualifiedName) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("domainExists"); - boolean exists = false; try { - List> mustClauseList = new ArrayList(); - mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", DATA_PRODUCT_ENTITY_TYPE))); - mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); - mustClauseList.add(mapOf("term", mapOf("name.keyword", productName))); - + exists(DATA_PRODUCT_ENTITY_TYPE, productName, parentDomainQualifiedName); - Map bool = new HashMap<>(); - if (parentDomain != null && StringUtils.isNotEmpty(parentDomainQualifiedName)){ - mustClauseList.add(mapOf("term", mapOf("parentDomainQualifiedName", parentDomainQualifiedName))); - } else { - List> mustNotClauseList = new ArrayList(); - mustNotClauseList.add(mapOf("exists", mapOf("field", "parentDomainQualifiedName"))); - bool.put("must_not", mustNotClauseList); - } - - bool.put("must", mustClauseList); - - Map dsl = mapOf("query", mapOf("bool", bool)); - - List products = indexSearchPaginated(dsl, null, this.discovery); - - if (CollectionUtils.isNotEmpty(products)) { - for (AtlasEntityHeader product : products) { - String name = (String) product.getAttribute(NAME); - if (productName.equals(name)) { - exists = true; - break; - } - } - } } finally { RequestContext.get().endMetricRecord(metricRecorder); } - - if (exists) { - throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, productName+" already exists in the domain"); - } } } From 77d90eabe5beac2461c565ac57f76b07d8b45d57 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Mon, 6 May 2024 21:30:03 +0530 Subject: [PATCH 135/316] DG-996 Fix delete 403 for Member user --- .../preprocessor/datamesh/AbstractDomainPreProcessor.java | 6 ------ 1 file changed, 6 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 4b491683fea..0d2d57b61c4 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -86,9 +86,6 @@ protected void isAuthorized(AtlasEntityHeader sourceDomain, AtlasEntityHeader ta AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, sourceDomain), "update on source Domain: ", sourceDomain.getAttribute(NAME)); - - AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_DELETE, sourceDomain), - "delete on source Domain: ", sourceDomain.getAttribute(NAME)); } if(targetDomain != null){ @@ -98,9 +95,6 @@ protected void isAuthorized(AtlasEntityHeader sourceDomain, AtlasEntityHeader ta AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, targetDomain), "update on target Domain: ", targetDomain.getAttribute(NAME)); - - AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_DELETE, targetDomain), - "delete on target Domain: ", targetDomain.getAttribute(NAME)); } } From 7f1f7c54947b22072e06f5e0ab45681778a745b1 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Tue, 7 May 2024 18:30:10 +0530 Subject: [PATCH 136/316] DG-1235 model, default titles, transformer --- addons/models/0000-Area0/0010-base_model.json | 20 +++++++++++++ .../policies/global_stakeholder-titles.json | 23 +++++++++++++++ .../policy_cache_transformer_persona.json | 29 +++++++++++++++++++ 3 files changed, 72 insertions(+) create mode 100644 addons/policies/global_stakeholder-titles.json diff --git a/addons/models/0000-Area0/0010-base_model.json b/addons/models/0000-Area0/0010-base_model.json index 4bfd6db9632..a6c51514db0 100644 --- a/addons/models/0000-Area0/0010-base_model.json +++ b/addons/models/0000-Area0/0010-base_model.json @@ -1017,6 +1017,26 @@ "isOptional": true, "isUnique": false, "includeInNotification": false + }, + { + "name": "domainGuid", + "typeName": "string", + "indexType": "STRING", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "includeInNotification": false + }, + { + "name": "stakeholderTitleGuid", + "typeName": "string", + "indexType": "STRING", + "cardinality": "SINGLE", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "includeInNotification": false } ] }, diff --git a/addons/policies/global_stakeholder-titles.json b/addons/policies/global_stakeholder-titles.json new file mode 100644 index 00000000000..65b2e098be3 --- /dev/null +++ b/addons/policies/global_stakeholder-titles.json @@ -0,0 +1,23 @@ +{ + "entities": [ + { + "typeName": "StakeholderTitle", + "attributes": + { + "qualifiedName": "stakeholderTitle/default/DOMAIN_OWNERS", + "name": "Domain Owners", + "domainGuid": "*" + } + }, + { + "typeName": "StakeholderTitle", + "attributes": + { + "qualifiedName": "stakeholderTitle/default/DATA_STEWARDS", + "name": "Data Stewards", + "domainGuid": "*" + } + } + ] +} + diff --git a/addons/static/templates/policy_cache_transformer_persona.json b/addons/static/templates/policy_cache_transformer_persona.json index a61b671090b..6fc53476ff7 100644 --- a/addons/static/templates/policy_cache_transformer_persona.json +++ b/addons/static/templates/policy_cache_transformer_persona.json @@ -404,6 +404,35 @@ "entity-remove-classification" ] }, + { + "policyResourceCategory": "ENTITY", + "policyType": "ACCESS", + "resources": [ + "entity:default/*/{entity}*", + "entity-type:Persona", + "entity-type:AuthPolicy", + "entity-classification:*" + ], + "actions": [ + "entity-create", + "entity-update", + "entity-delete" + ] + }, + { + "policyResourceCategory": "ENTITY", + "policyType": "ACCESS", + "resources": [ + "entity:stakeholderTitle/domain/{entity}/*", + "entity-type:StakeholderTitle", + "entity-classification:*" + ], + "actions": [ + "entity-create", + "entity-update", + "entity-delete" + ] + }, { "policyResourceCategory": "RELATIONSHIP", "policyType": "ACCESS", From bbeba9cc6423b9b24d9147cca2efbe39a88b1259 Mon Sep 17 00:00:00 2001 From: Rajat Movaliya Date: Wed, 8 May 2024 12:04:19 +0530 Subject: [PATCH 137/316] add traversalOrder for base node --- .../java/org/apache/atlas/discovery/EntityLineageService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index 5bdecd451fc..22f7039e6e4 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -303,6 +303,7 @@ private AtlasLineageOnDemandInfo getLineageInfoOnDemand(String guid, AtlasLineag traverseEdgesOnDemand(datasetVertex, false, depth, level, new HashSet<>(), atlasLineageOnDemandContext, ret, guid, outputEntitiesTraversed, traversalOrder); AtlasEntityHeader baseEntityHeader = entityRetriever.toAtlasEntityHeader(datasetVertex, atlasLineageOnDemandContext.getAttributes()); baseEntityHeader.setDepth(level); + baseEntityHeader.setTraversalOrder(0); ret.getGuidEntityMap().put(guid, baseEntityHeader); } else { AtlasVertex processVertex = AtlasGraphUtilsV2.findByGuid(this.graph, guid); From 64c29b17d1f1524e7f2eaeebb609620ba76347cf Mon Sep 17 00:00:00 2001 From: nikhilbonte21 Date: Wed, 8 May 2024 12:48:16 +0530 Subject: [PATCH 138/316] DG 1298 Form a QualifiedName for Domains and Products --- .../graph/v2/preprocessor/PreProcessorUtils.java | 2 ++ .../datamesh/AbstractDomainPreProcessor.java | 6 ++++++ .../datamesh/DataDomainPreProcessor.java | 12 ++++++++++++ .../datamesh/DataProductPreProcessor.java | 10 ++++++++++ 4 files changed, 30 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index c8728df251c..56e291ae85a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -53,6 +53,8 @@ public class PreProcessorUtils { public static final String PARENT_DOMAIN_QN_ATTR = "parentDomainQualifiedName"; public static final String SUPER_DOMAIN_QN_ATTR = "superDomainQualifiedName"; + public static final String MIGRATION_CUSTOM_ATTRIBUTE = "isQualifiedNameMigrated"; + //Query models constants public static final String PREFIX_QUERY_QN = "default/collection/"; public static final String COLLECTION_QUALIFIED_NAME = "collectionQualifiedName"; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 0d2d57b61c4..2d48e23787a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -65,6 +65,12 @@ public abstract class AbstractDomainPreProcessor implements PreProcessor { static final Set PARENT_ATTRIBUTES = new HashSet<>(Arrays.asList(SUPER_DOMAIN_QN_ATTR, PARENT_DOMAIN_QN_ATTR)); + static final Map customAttributes = new HashMap<>(); + + static { + customAttributes.put(MIGRATION_CUSTOM_ATTRIBUTE, "true"); + } + AbstractDomainPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, AtlasGraph graph) { this.entityRetriever = entityRetriever; this.typeRegistry = typeRegistry; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java index b30f6b3c3a6..8fcc32c3dd4 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java @@ -84,6 +84,7 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co private void processCreateDomain(AtlasEntity entity) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreateDomain"); String domainName = (String) entity.getAttribute(NAME); + String parentDomainQualifiedName = (String) entity.getAttribute(PARENT_DOMAIN_QN_ATTR); AtlasEntityHeader parentDomain = getParent(entity); @@ -91,6 +92,9 @@ private void processCreateDomain(AtlasEntity entity) throws AtlasBaseException { parentDomainQualifiedName = (String) parentDomain.getAttribute(QUALIFIED_NAME); } + entity.setAttribute(QUALIFIED_NAME, createQualifiedName(parentDomainQualifiedName)); + entity.setCustomAttributes(customAttributes); + domainExists(domainName, parentDomainQualifiedName); RequestContext.get().endMetricRecord(metricRecorder); @@ -307,6 +311,14 @@ private void domainExists(String domainName, String parentDomainQualifiedName) t RequestContext.get().endMetricRecord(metricRecorder); } } + + private static String createQualifiedName(String parentDomainQualifiedName) { + if (StringUtils.isNotEmpty(parentDomainQualifiedName)) { + return parentDomainQualifiedName + "/domain/" + getUUID(); + } else{ + return "default/domain/" + getUUID() + "/super"; + } + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 59048a498f2..babebd7c902 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -65,6 +65,9 @@ private void processCreateProduct(AtlasEntity entity) throws AtlasBaseException parentDomainQualifiedName = (String) parentDomain.getAttribute(QUALIFIED_NAME); } + entity.setAttribute(QUALIFIED_NAME, createQualifiedName(parentDomainQualifiedName)); + entity.setCustomAttributes(customAttributes); + productExists(productName, parentDomainQualifiedName); RequestContext.get().endMetricRecord(metricRecorder); @@ -179,4 +182,11 @@ private void productExists(String productName, String parentDomainQualifiedName) } } + private static String createQualifiedName(String parentDomainQualifiedName) throws AtlasBaseException { + if (StringUtils.isEmpty(parentDomainQualifiedName)) { + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Parent Domain Qualified Name cannot be empty or null"); + } + return parentDomainQualifiedName + "/product/" + getUUID(); + + } } From 06524eb0848ec6dc1bf9e11aba7a92b29649c9f7 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Wed, 8 May 2024 02:30:26 +0530 Subject: [PATCH 139/316] DG-1235 Preprocessors --- .../policy_cache_transformer_persona.json | 46 +++++- .../apache/atlas/repository/Constants.java | 2 + .../store/graph/v2/AtlasEntityStoreV2.java | 5 + .../graph/v2/preprocessor/PreProcessor.java | 2 + .../accesscontrol/PersonaPreProcessor.java | 83 +++++++++- .../StakeholderTitlePreProcessor.java | 147 ++++++++++++++++++ 6 files changed, 276 insertions(+), 9 deletions(-) create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java diff --git a/addons/static/templates/policy_cache_transformer_persona.json b/addons/static/templates/policy_cache_transformer_persona.json index 6fc53476ff7..88cdc6e7cf2 100644 --- a/addons/static/templates/policy_cache_transformer_persona.json +++ b/addons/static/templates/policy_cache_transformer_persona.json @@ -408,8 +408,21 @@ "policyResourceCategory": "ENTITY", "policyType": "ACCESS", "resources": [ - "entity:default/*/{entity}*", + "entity:default/*/{entity}", "entity-type:Persona", + "entity-classification:*" + ], + "actions": [ + "entity-create", + "entity-update", + "entity-delete" + ] + }, + { + "policyResourceCategory": "ENTITY", + "policyType": "ACCESS", + "resources": [ + "entity:default/*/{entity}/*", "entity-type:AuthPolicy", "entity-classification:*" ], @@ -423,7 +436,7 @@ "policyResourceCategory": "ENTITY", "policyType": "ACCESS", "resources": [ - "entity:stakeholderTitle/domain/{entity}/*", + "entity:stakeholderTitle/domain/*/{entity}", "entity-type:StakeholderTitle", "entity-classification:*" ], @@ -571,6 +584,35 @@ "entity-remove-classification" ] }, + { + "policyResourceCategory": "ENTITY", + "policyType": "ACCESS", + "resources": [ + "entity:default/*/{entity}/*", + "entity-type:Persona", + "entity-type:AuthPolicy", + "entity-classification:*" + ], + "actions": [ + "entity-create", + "entity-update", + "entity-delete" + ] + }, + { + "policyResourceCategory": "ENTITY", + "policyType": "ACCESS", + "resources": [ + "entity:stakeholderTitle/domain/*/{entity}/*domain/*", + "entity-type:StakeholderTitle", + "entity-classification:*" + ], + "actions": [ + "entity-create", + "entity-update", + "entity-delete" + ] + }, { "policyResourceCategory": "RELATIONSHIP", "policyType": "ACCESS", diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index b717a85229b..9ecfb46930c 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -431,6 +431,8 @@ public enum SupportedFileExtensions { XLSX, XLS, CSV } public static final Set SKIP_UPDATE_AUTH_CHECK_TYPES = new HashSet() {{ add(README_ENTITY_TYPE); add(LINK_ENTITY_TYPE); + add("Persona"); + add("StakeholderTitle"); }}; public static final Set SKIP_DELETE_AUTH_CHECK_TYPES = new HashSet() {{ diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 70fda21198b..956e065858c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -59,6 +59,7 @@ import org.apache.atlas.repository.store.graph.v2.preprocessor.AuthPolicyPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.ConnectionPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.contract.ContractPreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.datamesh.StakeholderTitlePreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.resource.LinkPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.accesscontrol.PersonaPreProcessor; @@ -1850,6 +1851,10 @@ public PreProcessor getPreProcessor(String typeName) { case CONTRACT_ENTITY_TYPE: preProcessor = new ContractPreProcessor(graph, typeRegistry, entityRetriever, storeDifferentialAudits, discovery); break; + + case "StakeholderTitle": + preProcessor = new StakeholderTitlePreProcessor(graph, typeRegistry, entityRetriever, this); + break; } return preProcessor; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessor.java index 0ed51910044..79119408f0c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessor.java @@ -18,6 +18,8 @@ public interface PreProcessor { Set skipInitialAuthCheckTypes = new HashSet() {{ add(ATLAS_GLOSSARY_TERM_ENTITY_TYPE); add(ATLAS_GLOSSARY_CATEGORY_ENTITY_TYPE); + add("Persona"); + add("StakeholderTitle"); }}; void processAttributes(AtlasStruct entity, EntityMutationContext context, EntityMutations.EntityOperation operation) throws AtlasBaseException; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java index 3541e3e4a72..986456f5ecd 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java @@ -19,9 +19,13 @@ import org.apache.atlas.RequestContext; +import org.apache.atlas.authorize.AtlasAuthorizationUtils; +import org.apache.atlas.authorize.AtlasEntityAccessRequest; +import org.apache.atlas.authorize.AtlasPrivilege; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.auth.client.keycloak.AtlasKeycloakClient; import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.instance.AtlasRelatedObjectId; import org.apache.atlas.model.instance.AtlasRelationship; @@ -41,6 +45,7 @@ import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; import org.keycloak.representations.idm.RoleRepresentation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -50,7 +55,9 @@ import java.util.List; import java.util.Map; +import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; import static org.apache.atlas.AtlasErrorCode.OPERATION_NOT_SUPPORTED; +import static org.apache.atlas.repository.Constants.NAME; import static org.apache.atlas.repository.Constants.POLICY_ENTITY_TYPE; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_ACCESS_CONTROL_ENABLED; @@ -138,31 +145,54 @@ public void processDelete(AtlasVertex vertex) throws AtlasBaseException { aliasStore.deleteAlias(getESAliasName(persona)); } - private void processCreatePersona(AtlasStruct entity) throws AtlasBaseException { + private void processCreatePersona(AtlasEntity entity) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreatePersona"); - validateNoPoliciesAttached((AtlasEntity) entity); + validateNoPoliciesAttached(entity); String tenantId = getTenantId(entity); - entity.setAttribute(QUALIFIED_NAME, String.format("%s/%s", tenantId, getUUID())); + if (entity.hasRelationshipAttribute("dataDomain")) { + if (! entity.hasRelationshipAttribute("stakeholderTitle")) { + throw new AtlasBaseException(BAD_REQUEST, "Relation stakeholderTitle not found"); + } + + entity.setAttribute("domainGuid", getGuidFromRelationAttribute(entity, "dataDomain")); + entity.setAttribute("stakeholderTitleGuid", getGuidFromRelationAttribute(entity, "stakeholderTitle")); + + String personaQualifiedName = String.format("default/%s/%s", + getUUID(), + getQualifiedNameFromRelationAttribute(entity, "dataDomain")); + + entity.setAttribute(QUALIFIED_NAME, personaQualifiedName); + + //TODO: validate Stakeholder & StakeholderTitle pair is unique + + } else { + + entity.setAttribute(QUALIFIED_NAME, String.format("%s/%s", tenantId, getUUID())); + } + + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, new AtlasEntityHeader(entity)), + "create Persona: ", entity.getAttribute(NAME)); + entity.setAttribute(ATTR_ACCESS_CONTROL_ENABLED, entity.getAttributes().getOrDefault(ATTR_ACCESS_CONTROL_ENABLED, true)); //create keycloak role - String roleId = createKeycloakRole((AtlasEntity) entity); + String roleId = createKeycloakRole(entity); entity.setAttribute(ATTR_PERSONA_ROLE_ID, roleId); //create ES alias - aliasStore.createAlias((AtlasEntity) entity); + aliasStore.createAlias(entity); RequestContext.get().endMetricRecord(metricRecorder); } - private void processUpdatePersona(EntityMutationContext context, AtlasStruct entity) throws AtlasBaseException { + private void processUpdatePersona(EntityMutationContext context, AtlasEntity entity) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdatePersona"); - AtlasEntity persona = (AtlasEntity) entity; + AtlasEntity persona = entity; validateNoPoliciesAttached(persona); AtlasVertex vertex = context.getVertex(persona.getGuid()); @@ -172,10 +202,21 @@ private void processUpdatePersona(EntityMutationContext context, AtlasStruct ent throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Persona not Active"); } + String domainGuid = (String) entity.getAttribute("domainGuid"); + if (StringUtils.isNotEmpty(domainGuid)) { + entity.removeAttribute("domainGuid"); + entity.removeAttribute("stakeholderTitleGuid"); + entity.getRelationshipAttributes().remove("dataDomain"); + entity.getRelationshipAttributes().remove("stakeholderTitle"); + } + String vertexQName = vertex.getProperty(QUALIFIED_NAME, String.class); entity.setAttribute(QUALIFIED_NAME, vertexQName); entity.setAttribute(ATTR_PERSONA_ROLE_ID, getPersonaRoleId(existingPersonaEntity)); + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, new AtlasEntityHeader(entity)), + "update Persona: ", entity.getAttribute(NAME)); + boolean isEnabled = getIsAccessControlEnabled(persona); if (getIsAccessControlEnabled(existingPersonaEntity) != isEnabled) { updatePoliciesIsEnabledAttr(context, existingPersonaEntity, isEnabled); @@ -263,4 +304,32 @@ private void updateKeycloakRole(AtlasEntity newPersona, AtlasEntity existingPers LOG.info("Updated keycloak role with name {}", roleName); } } + + private String getGuidFromRelationAttribute(AtlasEntity entity, String relationshipAttributeName) throws AtlasBaseException { + AtlasObjectId relationObjectId = (AtlasObjectId) entity.getRelationshipAttribute(relationshipAttributeName); + + String guid = relationObjectId.getGuid(); + if (StringUtils.isEmpty(guid)) { + AtlasVertex vertex = entityRetriever.getEntityVertex(relationObjectId); + guid = vertex.getProperty("__guid", String.class); + } + + return guid; + } + + private String getQualifiedNameFromRelationAttribute(AtlasEntity entity, String relationshipAttributeName) throws AtlasBaseException { + AtlasObjectId relationObjectId = (AtlasObjectId) entity.getRelationshipAttribute(relationshipAttributeName); + String qualifiedName = null; + + if (relationObjectId.getUniqueAttributes() != null) { + qualifiedName = (String) relationObjectId.getUniqueAttributes().get(QUALIFIED_NAME); + } + + if (StringUtils.isEmpty(qualifiedName)) { + AtlasVertex vertex = entityRetriever.getEntityVertex(relationObjectId); + qualifiedName = vertex.getProperty(QUALIFIED_NAME, String.class); + } + + return qualifiedName; + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java new file mode 100644 index 00000000000..5614a69f9d5 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java @@ -0,0 +1,147 @@ +package org.apache.atlas.repository.store.graph.v2.preprocessor.datamesh; + +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.RequestContext; +import org.apache.atlas.authorize.AtlasAuthorizationUtils; +import org.apache.atlas.authorize.AtlasEntityAccessRequest; +import org.apache.atlas.authorize.AtlasPrivilege; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.model.instance.AtlasObjectId; +import org.apache.atlas.model.instance.AtlasRelatedObjectId; +import org.apache.atlas.model.instance.AtlasRelationship; +import org.apache.atlas.model.instance.AtlasStruct; +import org.apache.atlas.model.instance.EntityMutations; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.AtlasEntityStore; +import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; +import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; +import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; +import org.apache.atlas.repository.store.users.KeycloakStore; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.List; +import java.util.Optional; + +import static org.apache.atlas.AtlasErrorCode.OPERATION_NOT_SUPPORTED; +import static org.apache.atlas.repository.Constants.NAME; +import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.getUUID; + +public class StakeholderTitlePreProcessor implements PreProcessor { + + private static final Logger LOG = LoggerFactory.getLogger(StakeholderTitlePreProcessor.class); + + private final AtlasGraph graph; + private final AtlasTypeRegistry typeRegistry; + private final EntityGraphRetriever entityRetriever; + private AtlasEntityStore entityStore; + + public StakeholderTitlePreProcessor(AtlasGraph graph, + AtlasTypeRegistry typeRegistry, + EntityGraphRetriever entityRetriever, + AtlasEntityStore entityStore) { + this.graph = graph; + this.typeRegistry = typeRegistry; + this.entityRetriever = entityRetriever; + this.entityStore = entityStore; + } + + @Override + public void processAttributes(AtlasStruct entityStruct, EntityMutationContext context, + EntityMutations.EntityOperation operation) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("StakeholderTitle.processAttributes: pre processing {}, {}", entityStruct.getAttribute(QUALIFIED_NAME), operation); + } + + AtlasEntity entity = (AtlasEntity) entityStruct; + + switch (operation) { + case CREATE: + processCreateStakeholderTitle(entity); + break; + case UPDATE: + processUpdateStakeholderTitle(context, entity); + break; + } + } + + private void processCreateStakeholderTitle(AtlasEntity entity) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreateStakeholderTitle"); + + try { + if (entity.hasRelationshipAttribute("stakeholders")) { + throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Can not attach a Stakeholder while creating StakeholderTitle"); + } + + String qualifiedName; + + String domainGuid = (String) entity.getAttribute("domainGuid"); + if ("*".equals(domainGuid)) { + qualifiedName = String.format("stakeholderTitle/domain/default/%s", getUUID()); + //TODO: validate name duplication + } else { + + AtlasVertex domain = entityRetriever.getEntityVertex(domainGuid); + qualifiedName = String.format("stakeholderTitle/domain/%s/%s", + getUUID(), + domain.getProperty(QUALIFIED_NAME, String.class)); + + //TODO: validate name duplication + } + + entity.setAttribute(QUALIFIED_NAME, qualifiedName); + + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, new AtlasEntityHeader(entity)), + "create StakeholderTitle: ", entity.getAttribute(NAME)); + + } finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + } + + private void processUpdateStakeholderTitle(EntityMutationContext context, AtlasEntity entity) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdateStakeholderTitle"); + + try { + if (entity.hasRelationshipAttribute("stakeholders")) { + throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Can not attach/detach a Stakeholder while updating StakeholderTitle"); + } + + AtlasVertex vertex = context.getVertex(entity.getGuid()); + String vertexQName = vertex.getProperty(QUALIFIED_NAME, String.class); + entity.setAttribute(QUALIFIED_NAME, vertexQName); + + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, new AtlasEntityHeader(entity)), + "update StakeholderTitle: ", entity.getAttribute(NAME)); + + } finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + } + + @Override + public void processDelete(AtlasVertex vertex) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processDeleteStakeholderTitle"); + + try { + AtlasEntity titleEntity = entityRetriever.toAtlasEntity(vertex); + List stakeholders = (List) titleEntity.getRelationshipAttribute("stakeholders"); + + Optional activeStakeholder = stakeholders.stream().filter(x -> x.getRelationshipStatus() == AtlasRelationship.Status.ACTIVE).findFirst(); + if (activeStakeholder.isPresent()) { + throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Can not delete StakeholderTitle as it has reference to Active Stakeholder"); + } + + } finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + } +} + From 3c66058ffa5bc1f91872b7df14e6113960ea11ad Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Wed, 8 May 2024 14:19:16 +0530 Subject: [PATCH 140/316] DG-1235 Fix default titles bloating issue --- .../StakeholderTitlePreProcessor.java | 34 ++++++++++--------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java index 5614a69f9d5..2ec047dc0ca 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java @@ -23,6 +23,7 @@ import org.apache.atlas.repository.store.users.KeycloakStore; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; +import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -80,27 +81,28 @@ private void processCreateStakeholderTitle(AtlasEntity entity) throws AtlasBaseE throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Can not attach a Stakeholder while creating StakeholderTitle"); } - String qualifiedName; + if (StringUtils.isNotEmpty(RequestContext.getCurrentUser())) { + String qualifiedName; - String domainGuid = (String) entity.getAttribute("domainGuid"); - if ("*".equals(domainGuid)) { - qualifiedName = String.format("stakeholderTitle/domain/default/%s", getUUID()); - //TODO: validate name duplication - } else { + String domainGuid = (String) entity.getAttribute("domainGuid"); + if ("*".equals(domainGuid)) { + qualifiedName = String.format("stakeholderTitle/domain/default/%s", getUUID()); + //TODO: validate name duplication + } else { - AtlasVertex domain = entityRetriever.getEntityVertex(domainGuid); - qualifiedName = String.format("stakeholderTitle/domain/%s/%s", - getUUID(), - domain.getProperty(QUALIFIED_NAME, String.class)); + AtlasVertex domain = entityRetriever.getEntityVertex(domainGuid); + qualifiedName = String.format("stakeholderTitle/domain/%s/%s", + getUUID(), + domain.getProperty(QUALIFIED_NAME, String.class)); - //TODO: validate name duplication - } - - entity.setAttribute(QUALIFIED_NAME, qualifiedName); + //TODO: validate name duplication + } - AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, new AtlasEntityHeader(entity)), - "create StakeholderTitle: ", entity.getAttribute(NAME)); + entity.setAttribute(QUALIFIED_NAME, qualifiedName); + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, new AtlasEntityHeader(entity)), + "create StakeholderTitle: ", entity.getAttribute(NAME)); + } } finally { RequestContext.get().endMetricRecord(metricRecorder); } From 9cb3abf619a6e210711183641f626916a86af3a7 Mon Sep 17 00:00:00 2001 From: Rajat Movaliya Date: Wed, 8 May 2024 14:35:08 +0530 Subject: [PATCH 141/316] add support for column lineage --- .../org/apache/atlas/discovery/EntityLineageService.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index 638523ad6d9..c9c11e4a862 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -1470,7 +1470,14 @@ private void processEdge(final AtlasEdge edge, final Map Date: Wed, 8 May 2024 22:02:12 +0530 Subject: [PATCH 142/316] Handle issue moving subDomain to make it Super Domain --- .../graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java | 1 + 1 file changed, 1 insertion(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java index 8fcc32c3dd4..f2bec8a2608 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java @@ -177,6 +177,7 @@ private void processMoveSubDomainToAnotherDomain(AtlasEntity domain, //Moving subDomain to make it Super Domain targetDomainQualifiedName = "default"; updatedQualifiedName = currentDomainQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); + updatedQualifiedName = updatedQualifiedName + "/super"; domain.setAttribute(QUALIFIED_NAME, updatedQualifiedName); domain.setAttribute(PARENT_DOMAIN_QN_ATTR, null); domain.setAttribute(SUPER_DOMAIN_QN_ATTR, null); From a27fdf608b10af33b6659889f48b112257e9ed8c Mon Sep 17 00:00:00 2001 From: Ujala Singh <59909223+ujala-singh@users.noreply.github.com> Date: Wed, 8 May 2024 22:10:57 +0530 Subject: [PATCH 143/316] feat: Add service PRs to repository dispatch (#2920) --- .../workflows/chart-release-dispatcher.yaml | 34 ++++++++++++++++++- 1 file changed, 33 insertions(+), 1 deletion(-) diff --git a/.github/workflows/chart-release-dispatcher.yaml b/.github/workflows/chart-release-dispatcher.yaml index 8e325cd3277..10fc6cbdb00 100644 --- a/.github/workflows/chart-release-dispatcher.yaml +++ b/.github/workflows/chart-release-dispatcher.yaml @@ -26,12 +26,42 @@ jobs: run: | echo "branch=${{ github.event.workflow_run.head_branch }}" >> $GITHUB_OUTPUT + # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it + - uses: actions/checkout@v3 + with: + token: ${{ secrets.my_pat }} + ref: ${{ steps.extract_branch.outputs.branch }} + fetch-depth: 0 + + - name: Get SHA of the branch + id: get_sha + run: | + branch_name=${{ steps.extract_branch.outputs.branch }} + sha=$(git rev-parse "refs/heads/$branch_name") + echo "GIT_SHA: $sha" + echo "sha=${sha}" >> $GITHUB_OUTPUT + - name: Extract Repository Name id: extract_repo_name run: | repo_name=$(basename $GITHUB_REPOSITORY) echo "repo_name=${repo_name}" >> $GITHUB_OUTPUT + - name: Get PR url and PR User + id: get_pr_url_user + run: | + head_sha=$(curl -s -H "Authorization: Bearer ${{ secrets.my_pat }}" -H "Accept: application/vnd.github.v3+json" "https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }}/jobs" | jq -r '.jobs[0].head_sha') + echo "Head SHA: $head_sha" + pr_url=$(curl -s -H "Authorization: Bearer ${{ secrets.my_pat }}" -H "Accept: application/vnd.github.v3+json" "https://api.github.com/search/issues?q=sha:$head_sha+type:pr" | jq -r '.items[0].html_url') + pr_user=$(curl -s -H "Authorization: Bearer ${{ secrets.my_pat }}" -H "Accept: application/vnd.github.v3+json" "https://api.github.com/search/issues?q=sha:$head_sha+type:pr" | jq -r '.items[0].user.login') + echo "pr_url=$pr_url" >> $GITHUB_OUTPUT + echo "pr_user=$pr_user" >> $GITHUB_OUTPUT + + - name: echo PR_URL and PR_USER + run: | + echo "${{ steps.get_pr_url_user.outputs.pr_url }}" + echo "${{ steps.get_pr_url_user.outputs.pr_user }}" + - name: Repository Dispatch uses: peter-evans/repository-dispatch@v2 with: @@ -42,6 +72,8 @@ jobs: { "repo": { "name": "${{ steps.extract_repo_name.outputs.repo_name }}", - "branch": "${{ steps.extract_branch.outputs.branch }}" + "branch": "${{ steps.extract_branch.outputs.branch }}", + "pr_url": "${{ steps.get_pr_url_user.outputs.pr_url }}", + "pr_user": "${{ steps.get_pr_url_user.outputs.pr_user }}" } } From 95da7fd404f179e94e4f1c59f401ccce5442252e Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Thu, 9 May 2024 00:23:28 +0530 Subject: [PATCH 144/316] DG 1298 Handle issue moving subDomain to make it Super Domain --- .../datamesh/DataDomainPreProcessor.java | 30 +++++++++++++++++-- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java index f2bec8a2608..ea09fb69470 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java @@ -190,7 +190,7 @@ private void processMoveSubDomainToAnotherDomain(AtlasEntity domain, domain.setAttribute(SUPER_DOMAIN_QN_ATTR, superDomainQualifiedName); } - moveChildrenToAnotherDomain(domainVertex, superDomainQualifiedName, null, sourceDomainQualifiedName, targetDomainQualifiedName); + moveChildren(domainVertex, superDomainQualifiedName, updatedQualifiedName, sourceDomainQualifiedName, targetDomainQualifiedName); updatePolicies(this.updatedPolicyResources, this.context); LOG.info("Moved subDomain {} to Domain {}", domainName, targetDomainQualifiedName); @@ -200,6 +200,25 @@ private void processMoveSubDomainToAnotherDomain(AtlasEntity domain, } } + private void moveChildren(AtlasVertex domainVertex, + String superDomainQualifiedName, + String parentDomainQualifiedName, + String sourceDomainQualifiedName, + String targetDomainQualifiedName) throws AtlasBaseException { + // move products to target Domain + Iterator products = getActiveChildrenVertices(domainVertex, DATA_PRODUCT_EDGE_LABEL); + while (products.hasNext()) { + AtlasVertex productVertex = products.next(); + moveChildDataProductToAnotherDomain(productVertex, superDomainQualifiedName, parentDomainQualifiedName, sourceDomainQualifiedName, targetDomainQualifiedName); + } + // Get all children domains of current domain + Iterator childDomains = getActiveChildrenVertices(domainVertex, DOMAIN_PARENT_EDGE_LABEL); + while (childDomains.hasNext()) { + AtlasVertex childVertex = childDomains.next(); + moveChildrenToAnotherDomain(childVertex, superDomainQualifiedName, parentDomainQualifiedName, sourceDomainQualifiedName, targetDomainQualifiedName); + } + } + private void moveChildrenToAnotherDomain(AtlasVertex childDomainVertex, String superDomainQualifiedName, String parentDomainQualifiedName, @@ -213,7 +232,7 @@ private void moveChildrenToAnotherDomain(AtlasVertex childDomainVertex, Map updatedAttributes = new HashMap<>(); String currentDomainQualifiedName = childDomainVertex.getProperty(QUALIFIED_NAME, String.class); - String updatedDomainQualifiedName = currentDomainQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); + String updatedDomainQualifiedName = parentDomainQualifiedName + getOwnQualifiedNameForChild(currentDomainQualifiedName); // Change domain qualifiedName childDomainVertex.setProperty(QUALIFIED_NAME, updatedDomainQualifiedName); @@ -269,7 +288,7 @@ private void moveChildDataProductToAnotherDomain(AtlasVertex productVertex, Map updatedAttributes = new HashMap<>(); String currentQualifiedName = productVertex.getProperty(QUALIFIED_NAME, String.class); - String updatedQualifiedName = currentQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); + String updatedQualifiedName = parentDomainQualifiedName + getOwnQualifiedNameForChild(currentQualifiedName); productVertex.setProperty(QUALIFIED_NAME, updatedQualifiedName); updatedAttributes.put(QUALIFIED_NAME, updatedQualifiedName); @@ -320,6 +339,11 @@ private static String createQualifiedName(String parentDomainQualifiedName) { return "default/domain/" + getUUID() + "/super"; } } + + private String getOwnQualifiedNameForChild(String childQualifiedName) { + String[] splitted = childQualifiedName.split("/"); + return String.format("/%s/%s", splitted[splitted.length -2], splitted[splitted.length -1]); + } } From 19217b21aff92713c69806cae7eb92aa0e580e09 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Thu, 9 May 2024 01:13:46 +0530 Subject: [PATCH 145/316] Logging --- .../preprocessor/datamesh/AbstractDomainPreProcessor.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 2d48e23787a..796317a904e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -37,6 +37,7 @@ import org.apache.atlas.repository.store.graph.v2.preprocessor.AuthPolicyPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; import org.apache.atlas.type.AtlasEntityType; +import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; @@ -109,10 +110,15 @@ protected void updatePolicies(Map updatedPolicyResources, Entity try { AtlasEntityType entityType = typeRegistry.getEntityTypeByName(POLICY_ENTITY_TYPE); + LOG.info("updatedPolicyResources.size: {} ", updatedPolicyResources.size()); + LOG.info("updatedPolicyResources {} ", AtlasType.toJson(updatedPolicyResources)); + List policies = getPolicies(updatedPolicyResources.keySet()); + LOG.info("policies.size: {} ", policies.size()); if (CollectionUtils.isNotEmpty(policies)) { for (AtlasEntityHeader policy : policies) { + LOG.info("Updating policy {} ", policy.getGuid()); AtlasVertex policyVertex = entityRetriever.getEntityVertex(policy.getGuid()); AtlasEntity policyEntity = entityRetriever.toAtlasEntity(policyVertex); From dbdccec5a0eb3c0e823c738e8978bc9badfeaa6d Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Thu, 9 May 2024 01:31:15 +0530 Subject: [PATCH 146/316] Fix policy not updating issue --- .../datamesh/AbstractDomainPreProcessor.java | 11 ++++++----- .../preprocessor/datamesh/DataDomainPreProcessor.java | 5 ++++- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 796317a904e..936ba6d948e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -37,10 +37,10 @@ import org.apache.atlas.repository.store.graph.v2.preprocessor.AuthPolicyPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; import org.apache.atlas.type.AtlasEntityType; -import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -110,15 +110,16 @@ protected void updatePolicies(Map updatedPolicyResources, Entity try { AtlasEntityType entityType = typeRegistry.getEntityTypeByName(POLICY_ENTITY_TYPE); - LOG.info("updatedPolicyResources.size: {} ", updatedPolicyResources.size()); - LOG.info("updatedPolicyResources {} ", AtlasType.toJson(updatedPolicyResources)); + if (MapUtils.isEmpty(updatedPolicyResources)) { + return; + } List policies = getPolicies(updatedPolicyResources.keySet()); - LOG.info("policies.size: {} ", policies.size()); + LOG.info("Found {} policies to update", policies.size()); if (CollectionUtils.isNotEmpty(policies)) { for (AtlasEntityHeader policy : policies) { - LOG.info("Updating policy {} ", policy.getGuid()); + LOG.info("Updating Policy {}", policy.getGuid()); AtlasVertex policyVertex = entityRetriever.getEntityVertex(policy.getGuid()); AtlasEntity policyEntity = entityRetriever.toAtlasEntity(policyVertex); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java index ea09fb69470..f17a36e8332 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java @@ -190,6 +190,9 @@ private void processMoveSubDomainToAnotherDomain(AtlasEntity domain, domain.setAttribute(SUPER_DOMAIN_QN_ATTR, superDomainQualifiedName); } + String currentQualifiedName = domainVertex.getProperty(QUALIFIED_NAME, String.class); + this.updatedPolicyResources.put("entity:" + currentQualifiedName, "entity:" + updatedQualifiedName); + moveChildren(domainVertex, superDomainQualifiedName, updatedQualifiedName, sourceDomainQualifiedName, targetDomainQualifiedName); updatePolicies(this.updatedPolicyResources, this.context); @@ -339,7 +342,7 @@ private static String createQualifiedName(String parentDomainQualifiedName) { return "default/domain/" + getUUID() + "/super"; } } - + private String getOwnQualifiedNameForChild(String childQualifiedName) { String[] splitted = childQualifiedName.split("/"); return String.format("/%s/%s", splitted[splitted.length -2], splitted[splitted.length -1]); From 6bf8a4009b6f2161abc1f83cf8df2d432ee05021 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Thu, 9 May 2024 10:25:34 +0530 Subject: [PATCH 147/316] DQ-161: Fix IndexOutOfBoundsException when removing updated or created entities from context --- .../atlas/repository/store/graph/v2/AtlasEntityStoreV2.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 70fda21198b..59afc180283 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -1538,8 +1538,7 @@ private void executePreProcessor(EntityMutationContext context) throws AtlasBase PreProcessor preProcessor; List copyOfCreated = new ArrayList<>(context.getCreatedEntities()); - for (int i = 0; i < copyOfCreated.size() ; i++) { - AtlasEntity entity = ((List) context.getCreatedEntities()).get(i); + for (AtlasEntity entity : copyOfCreated) { entityType = context.getType(entity.getGuid()); preProcessor = getPreProcessor(entityType.getTypeName()); @@ -1549,8 +1548,7 @@ private void executePreProcessor(EntityMutationContext context) throws AtlasBase } List copyOfUpdated = new ArrayList<>(context.getUpdatedEntities()); - for (int i = 0; i < copyOfUpdated.size() ; i++) { - AtlasEntity entity = ((List) context.getUpdatedEntities()).get(i); + for (AtlasEntity entity: copyOfUpdated) { entityType = context.getType(entity.getGuid()); preProcessor = getPreProcessor(entityType.getTypeName()); From 7072ce8f16f6fef920aa91df48b4d176ff7e4ab9 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Thu, 9 May 2024 12:35:21 +0530 Subject: [PATCH 148/316] DG-1235 Persona type change --- addons/models/0000-Area0/0010-base_model.json | 4 +- .../accesscontrol/PersonaPreProcessor.java | 5 ++- .../StakeholderTitlePreProcessor.java | 41 +++++++++++-------- 3 files changed, 29 insertions(+), 21 deletions(-) diff --git a/addons/models/0000-Area0/0010-base_model.json b/addons/models/0000-Area0/0010-base_model.json index a6c51514db0..2a29f072ef9 100644 --- a/addons/models/0000-Area0/0010-base_model.json +++ b/addons/models/0000-Area0/0010-base_model.json @@ -984,7 +984,7 @@ ], "description": "Atlan Type representing a Persona model", "serviceType": "atlan", - "typeVersion": "1.1", + "typeVersion": "1.2", "attributeDefs": [ { "name": "personaGroups", @@ -1019,7 +1019,7 @@ "includeInNotification": false }, { - "name": "domainGuid", + "name": "domainQualifiedName", "typeName": "string", "indexType": "STRING", "cardinality": "SINGLE", diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java index 986456f5ecd..4b47d9853aa 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java @@ -157,12 +157,13 @@ private void processCreatePersona(AtlasEntity entity) throws AtlasBaseException throw new AtlasBaseException(BAD_REQUEST, "Relation stakeholderTitle not found"); } - entity.setAttribute("domainGuid", getGuidFromRelationAttribute(entity, "dataDomain")); + String domainQualifiedName = getQualifiedNameFromRelationAttribute(entity, "dataDomain"); + entity.setAttribute("domainQualifiedName", domainQualifiedName); entity.setAttribute("stakeholderTitleGuid", getGuidFromRelationAttribute(entity, "stakeholderTitle")); String personaQualifiedName = String.format("default/%s/%s", getUUID(), - getQualifiedNameFromRelationAttribute(entity, "dataDomain")); + domainQualifiedName); entity.setAttribute(QUALIFIED_NAME, personaQualifiedName); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java index 2ec047dc0ca..b35840af133 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java @@ -81,28 +81,31 @@ private void processCreateStakeholderTitle(AtlasEntity entity) throws AtlasBaseE throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Can not attach a Stakeholder while creating StakeholderTitle"); } - if (StringUtils.isNotEmpty(RequestContext.getCurrentUser())) { - String qualifiedName; - - String domainGuid = (String) entity.getAttribute("domainGuid"); - if ("*".equals(domainGuid)) { - qualifiedName = String.format("stakeholderTitle/domain/default/%s", getUUID()); - //TODO: validate name duplication - } else { + if (RequestContext.get().isSkipAuthorizationCheck()) { + return; + } - AtlasVertex domain = entityRetriever.getEntityVertex(domainGuid); - qualifiedName = String.format("stakeholderTitle/domain/%s/%s", - getUUID(), - domain.getProperty(QUALIFIED_NAME, String.class)); + String qualifiedName; - //TODO: validate name duplication - } + String domainGuid = (String) entity.getAttribute("domainGuid"); + if ("*".equals(domainGuid)) { + qualifiedName = String.format("stakeholderTitle/domain/default/%s", getUUID()); + //TODO: validate name duplication + } else { - entity.setAttribute(QUALIFIED_NAME, qualifiedName); + AtlasVertex domain = entityRetriever.getEntityVertex(domainGuid); + qualifiedName = String.format("stakeholderTitle/domain/%s/%s", + getUUID(), + domain.getProperty(QUALIFIED_NAME, String.class)); - AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, new AtlasEntityHeader(entity)), - "create StakeholderTitle: ", entity.getAttribute(NAME)); + //TODO: validate name duplication } + + entity.setAttribute(QUALIFIED_NAME, qualifiedName); + + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, new AtlasEntityHeader(entity)), + "create StakeholderTitle: ", entity.getAttribute(NAME)); + } finally { RequestContext.get().endMetricRecord(metricRecorder); } @@ -112,6 +115,10 @@ private void processUpdateStakeholderTitle(EntityMutationContext context, AtlasE AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdateStakeholderTitle"); try { + if (RequestContext.get().isSkipAuthorizationCheck()) { + return; + } + if (entity.hasRelationshipAttribute("stakeholders")) { throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Can not attach/detach a Stakeholder while updating StakeholderTitle"); } From 65d80f07e56ca5fa77283ed230530e4094ab986d Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Thu, 9 May 2024 14:52:38 +0530 Subject: [PATCH 149/316] DG-1235 Fix persona alias extraction issue --- .../atlas/discovery/EntityDiscoveryService.java | 4 ++-- .../atlas/repository/util/AccessControlUtils.java | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index f2dab34a1a5..66e8d589091 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -44,6 +44,7 @@ import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.repository.userprofile.UserProfileService; +import org.apache.atlas.repository.util.AccessControlUtils; import org.apache.atlas.searchlog.ESSearchLogger; import org.apache.atlas.service.FeatureFlagStore; import org.apache.atlas.stats.StatsClient; @@ -1146,8 +1147,7 @@ private String getIndexName(IndexSearchParams params) throws AtlasBaseException qualifiedName = params.getPurpose(); } - String[] parts = qualifiedName.split("/"); - String aliasName = parts[parts.length - 1]; + String aliasName = AccessControlUtils.getESAliasName(qualifiedName); if (StringUtils.isNotEmpty(aliasName)) { if(params.isAccessControlExclusive()) { diff --git a/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java b/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java index 4588bdd84d5..dd2a035c6b5 100644 --- a/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java @@ -32,6 +32,7 @@ import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.util.NanoIdUtils; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -252,19 +253,18 @@ public static AtlasEntity extractConnectionFromResource(EntityGraphRetriever ent } public static String getPersonaRoleName(AtlasEntity persona) { - String qualifiedName = getStringAttribute(persona, QUALIFIED_NAME); - - String[] parts = qualifiedName.split("/"); - - return "persona_" + parts[parts.length - 1]; + return "persona_" + getESAliasName(persona); } public static String getESAliasName(AtlasEntity entity) { String qualifiedName = getStringAttribute(entity, QUALIFIED_NAME); + return getESAliasName(qualifiedName); + } + public static String getESAliasName(String qualifiedName) { String[] parts = qualifiedName.split("/"); - return parts[parts.length - 1]; + return parts[1]; } public static List getPolicies(AtlasEntity.AtlasEntityWithExtInfo accessControl) { From 594b682eba980527193d4f155dcc07845c13d5f8 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Thu, 9 May 2024 19:16:44 +0530 Subject: [PATCH 150/316] DG-1235 Stakeholder preprocessor --- addons/models/0000-Area0/0010-base_model.json | 45 ++-- .../policy_cache_transformer_persona.json | 52 +++- .../apache/atlas/repository/Constants.java | 1 + .../store/graph/v2/AtlasEntityStoreV2.java | 5 + .../accesscontrol/PersonaPreProcessor.java | 100 ++------ .../StakeholderPreProcessor.java | 231 ++++++++++++++++++ .../datamesh/DataDomainPreProcessor.java | 4 + .../StakeholderTitlePreProcessor.java | 4 +- .../repository/util/AccessControlUtils.java | 2 +- 9 files changed, 323 insertions(+), 121 deletions(-) create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java diff --git a/addons/models/0000-Area0/0010-base_model.json b/addons/models/0000-Area0/0010-base_model.json index 2a29f072ef9..5f21227e3dc 100644 --- a/addons/models/0000-Area0/0010-base_model.json +++ b/addons/models/0000-Area0/0010-base_model.json @@ -984,7 +984,7 @@ ], "description": "Atlan Type representing a Persona model", "serviceType": "atlan", - "typeVersion": "1.2", + "typeVersion": "1.1", "attributeDefs": [ { "name": "personaGroups", @@ -1017,26 +1017,6 @@ "isOptional": true, "isUnique": false, "includeInNotification": false - }, - { - "name": "domainQualifiedName", - "typeName": "string", - "indexType": "STRING", - "cardinality": "SINGLE", - "isIndexable": false, - "isOptional": true, - "isUnique": false, - "includeInNotification": false - }, - { - "name": "stakeholderTitleGuid", - "typeName": "string", - "indexType": "STRING", - "cardinality": "SINGLE", - "isIndexable": false, - "isOptional": true, - "isUnique": false, - "includeInNotification": false } ] }, @@ -1061,6 +1041,29 @@ "includeInNotification": false } ] + }, + { + "name": "StakeholderTitle", + "description": "Instance of a stakeholder title for Domains in Atlan", + "superTypes": [ + "Catalog" + ], + "serviceType": "atlan", + "typeVersion": "1.0", + "attributeDefs": [ + { + "name": "domainQualifiedNames", + "description": "qualified name array representing the Domains for which this StakeholderTitle is applicable", + "typeName": "array", + "indexType": "STRING", + "isOptional": true, + "cardinality": "SET", + "isUnique": false, + "isIndexable": false, + "skipScrubbing": true, + "includeInNotification": false + } + ] } ], "relationshipDefs": [ diff --git a/addons/static/templates/policy_cache_transformer_persona.json b/addons/static/templates/policy_cache_transformer_persona.json index 88cdc6e7cf2..2ae85d577d8 100644 --- a/addons/static/templates/policy_cache_transformer_persona.json +++ b/addons/static/templates/policy_cache_transformer_persona.json @@ -406,10 +406,11 @@ }, { "policyResourceCategory": "ENTITY", + "description": "Create Stakeholder for this Domain", "policyType": "ACCESS", "resources": [ "entity:default/*/{entity}", - "entity-type:Persona", + "entity-type:Stakeholder", "entity-classification:*" ], "actions": [ @@ -419,22 +420,28 @@ ] }, { - "policyResourceCategory": "ENTITY", + "policyResourceCategory": "RELATIONSHIP", "policyType": "ACCESS", + "description": "Link/unlink any Stakeholder to this Domain", + "resources": [ - "entity:default/*/{entity}/*", - "entity-type:AuthPolicy", - "entity-classification:*" + "relationship-type:*", + + "end-one-entity-type:DataDomain", + "end-one-entity-classification:*", + "end-one-entity:{entity}", + + "end-two-entity-type:Stakeholder", + "end-two-entity-classification:*", + "end-two-entity:default/*/{entity}" ], - "actions": [ - "entity-create", - "entity-update", - "entity-delete" - ] + "actions": ["add-relationship", "update-relationship", "remove-relationship"] }, { "policyResourceCategory": "ENTITY", "policyType": "ACCESS", + "description": "Create Stakeholder Title for this Domain", + "resources": [ "entity:stakeholderTitle/domain/*/{entity}", "entity-type:StakeholderTitle", @@ -587,10 +594,11 @@ { "policyResourceCategory": "ENTITY", "policyType": "ACCESS", + "description": "Create Stakeholder for Sub Domains", + "resources": [ "entity:default/*/{entity}/*", - "entity-type:Persona", - "entity-type:AuthPolicy", + "entity-type:Stakeholder", "entity-classification:*" ], "actions": [ @@ -599,9 +607,29 @@ "entity-delete" ] }, + { + "policyResourceCategory": "RELATIONSHIP", + "policyType": "ACCESS", + "description": "Link/unlink Stakeholder to Sub Domains", + + "resources": [ + "relationship-type:*", + + "end-one-entity-type:DataDomain", + "end-one-entity-classification:*", + "end-one-entity:{entity}/*domain/*", + + "end-two-entity-type:Stakeholder", + "end-two-entity-classification:*", + "end-two-entity:default/*/{entity}/*" + ], + "actions": ["add-relationship", "update-relationship", "remove-relationship"] + }, { "policyResourceCategory": "ENTITY", "policyType": "ACCESS", + "description": "Create Stakeholder Title for Sub Domains", + "resources": [ "entity:stakeholderTitle/domain/*/{entity}/*domain/*", "entity-type:StakeholderTitle", diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 9ecfb46930c..c8c68b87faf 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -158,6 +158,7 @@ public final class Constants { public static final String PURPOSE_ENTITY_TYPE = "Purpose"; public static final String POLICY_ENTITY_TYPE = "AuthPolicy"; public static final String SERVICE_ENTITY_TYPE = "AuthService"; + public static final String STAKEHOLDER_ENTITY_TYPE = "Stakeholder"; /** * Resource diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 956e065858c..70b76120d00 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -58,6 +58,7 @@ import org.apache.atlas.repository.store.graph.v1.RestoreHandlerV1; import org.apache.atlas.repository.store.graph.v2.preprocessor.AuthPolicyPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.ConnectionPreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.accesscontrol.StakeholderPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.contract.ContractPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.datamesh.StakeholderTitlePreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.resource.LinkPreProcessor; @@ -1836,6 +1837,10 @@ public PreProcessor getPreProcessor(String typeName) { preProcessor = new AuthPolicyPreProcessor(graph, typeRegistry, entityRetriever); break; + case STAKEHOLDER_ENTITY_TYPE: + preProcessor = new StakeholderPreProcessor(graph, typeRegistry, entityRetriever, this); + break; + case CONNECTION_ENTITY_TYPE: preProcessor = new ConnectionPreProcessor(graph, discovery, entityRetriever, featureFlagStore, deleteDelegate, this); break; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java index 4b47d9853aa..5c93b521d72 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java @@ -19,13 +19,9 @@ import org.apache.atlas.RequestContext; -import org.apache.atlas.authorize.AtlasAuthorizationUtils; -import org.apache.atlas.authorize.AtlasEntityAccessRequest; -import org.apache.atlas.authorize.AtlasPrivilege; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.auth.client.keycloak.AtlasKeycloakClient; import org.apache.atlas.model.instance.AtlasEntity; -import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.instance.AtlasRelatedObjectId; import org.apache.atlas.model.instance.AtlasRelationship; @@ -45,7 +41,6 @@ import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.StringUtils; import org.keycloak.representations.idm.RoleRepresentation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -55,9 +50,7 @@ import java.util.List; import java.util.Map; -import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; import static org.apache.atlas.AtlasErrorCode.OPERATION_NOT_SUPPORTED; -import static org.apache.atlas.repository.Constants.NAME; import static org.apache.atlas.repository.Constants.POLICY_ENTITY_TYPE; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_ACCESS_CONTROL_ENABLED; @@ -80,12 +73,12 @@ public class PersonaPreProcessor implements PreProcessor { private static final Logger LOG = LoggerFactory.getLogger(PersonaPreProcessor.class); - private final AtlasGraph graph; - private final AtlasTypeRegistry typeRegistry; - private final EntityGraphRetriever entityRetriever; - private IndexAliasStore aliasStore; - private AtlasEntityStore entityStore; - private KeycloakStore keycloakStore; + protected final AtlasGraph graph; + protected AtlasTypeRegistry typeRegistry; + protected final EntityGraphRetriever entityRetriever; + protected IndexAliasStore aliasStore; + protected AtlasEntityStore entityStore; + protected KeycloakStore keycloakStore; public PersonaPreProcessor(AtlasGraph graph, AtlasTypeRegistry typeRegistry, @@ -145,55 +138,31 @@ public void processDelete(AtlasVertex vertex) throws AtlasBaseException { aliasStore.deleteAlias(getESAliasName(persona)); } - private void processCreatePersona(AtlasEntity entity) throws AtlasBaseException { + private void processCreatePersona(AtlasStruct entity) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreatePersona"); - validateNoPoliciesAttached(entity); + validateNoPoliciesAttached((AtlasEntity) entity); String tenantId = getTenantId(entity); - if (entity.hasRelationshipAttribute("dataDomain")) { - if (! entity.hasRelationshipAttribute("stakeholderTitle")) { - throw new AtlasBaseException(BAD_REQUEST, "Relation stakeholderTitle not found"); - } - - String domainQualifiedName = getQualifiedNameFromRelationAttribute(entity, "dataDomain"); - entity.setAttribute("domainQualifiedName", domainQualifiedName); - entity.setAttribute("stakeholderTitleGuid", getGuidFromRelationAttribute(entity, "stakeholderTitle")); - - String personaQualifiedName = String.format("default/%s/%s", - getUUID(), - domainQualifiedName); - - entity.setAttribute(QUALIFIED_NAME, personaQualifiedName); - - //TODO: validate Stakeholder & StakeholderTitle pair is unique - - } else { - - entity.setAttribute(QUALIFIED_NAME, String.format("%s/%s", tenantId, getUUID())); - } - - AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, new AtlasEntityHeader(entity)), - "create Persona: ", entity.getAttribute(NAME)); - + entity.setAttribute(QUALIFIED_NAME, String.format("%s/%s", tenantId, getUUID())); entity.setAttribute(ATTR_ACCESS_CONTROL_ENABLED, entity.getAttributes().getOrDefault(ATTR_ACCESS_CONTROL_ENABLED, true)); //create keycloak role - String roleId = createKeycloakRole(entity); + String roleId = createKeycloakRole((AtlasEntity) entity); entity.setAttribute(ATTR_PERSONA_ROLE_ID, roleId); //create ES alias - aliasStore.createAlias(entity); + aliasStore.createAlias((AtlasEntity) entity); RequestContext.get().endMetricRecord(metricRecorder); } - private void processUpdatePersona(EntityMutationContext context, AtlasEntity entity) throws AtlasBaseException { + private void processUpdatePersona(EntityMutationContext context, AtlasStruct entity) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdatePersona"); - AtlasEntity persona = entity; + AtlasEntity persona = (AtlasEntity) entity; validateNoPoliciesAttached(persona); AtlasVertex vertex = context.getVertex(persona.getGuid()); @@ -203,21 +172,10 @@ private void processUpdatePersona(EntityMutationContext context, AtlasEntity ent throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Persona not Active"); } - String domainGuid = (String) entity.getAttribute("domainGuid"); - if (StringUtils.isNotEmpty(domainGuid)) { - entity.removeAttribute("domainGuid"); - entity.removeAttribute("stakeholderTitleGuid"); - entity.getRelationshipAttributes().remove("dataDomain"); - entity.getRelationshipAttributes().remove("stakeholderTitle"); - } - String vertexQName = vertex.getProperty(QUALIFIED_NAME, String.class); entity.setAttribute(QUALIFIED_NAME, vertexQName); entity.setAttribute(ATTR_PERSONA_ROLE_ID, getPersonaRoleId(existingPersonaEntity)); - AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, new AtlasEntityHeader(entity)), - "update Persona: ", entity.getAttribute(NAME)); - boolean isEnabled = getIsAccessControlEnabled(persona); if (getIsAccessControlEnabled(existingPersonaEntity) != isEnabled) { updatePoliciesIsEnabledAttr(context, existingPersonaEntity, isEnabled); @@ -251,7 +209,7 @@ private void updatePoliciesIsEnabledAttr(EntityMutationContext context, AtlasEnt } } - private String createKeycloakRole(AtlasEntity entity) throws AtlasBaseException { + protected String createKeycloakRole(AtlasEntity entity) throws AtlasBaseException { String roleName = getPersonaRoleName(entity); List users = getPersonaUsers(entity); List groups = getPersonaGroups(entity); @@ -270,7 +228,7 @@ private String createKeycloakRole(AtlasEntity entity) throws AtlasBaseException return role.getId(); } - private void updateKeycloakRole(AtlasEntity newPersona, AtlasEntity existingPersona) throws AtlasBaseException { + protected void updateKeycloakRole(AtlasEntity newPersona, AtlasEntity existingPersona) throws AtlasBaseException { String roleId = getPersonaRoleId(existingPersona); String roleName = getPersonaRoleName(existingPersona); @@ -305,32 +263,4 @@ private void updateKeycloakRole(AtlasEntity newPersona, AtlasEntity existingPers LOG.info("Updated keycloak role with name {}", roleName); } } - - private String getGuidFromRelationAttribute(AtlasEntity entity, String relationshipAttributeName) throws AtlasBaseException { - AtlasObjectId relationObjectId = (AtlasObjectId) entity.getRelationshipAttribute(relationshipAttributeName); - - String guid = relationObjectId.getGuid(); - if (StringUtils.isEmpty(guid)) { - AtlasVertex vertex = entityRetriever.getEntityVertex(relationObjectId); - guid = vertex.getProperty("__guid", String.class); - } - - return guid; - } - - private String getQualifiedNameFromRelationAttribute(AtlasEntity entity, String relationshipAttributeName) throws AtlasBaseException { - AtlasObjectId relationObjectId = (AtlasObjectId) entity.getRelationshipAttribute(relationshipAttributeName); - String qualifiedName = null; - - if (relationObjectId.getUniqueAttributes() != null) { - qualifiedName = (String) relationObjectId.getUniqueAttributes().get(QUALIFIED_NAME); - } - - if (StringUtils.isEmpty(qualifiedName)) { - AtlasVertex vertex = entityRetriever.getEntityVertex(relationObjectId); - qualifiedName = vertex.getProperty(QUALIFIED_NAME, String.class); - } - - return qualifiedName; - } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java new file mode 100644 index 00000000000..b499a274ce8 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java @@ -0,0 +1,231 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.store.graph.v2.preprocessor.accesscontrol; + + +import org.apache.atlas.RequestContext; +import org.apache.atlas.auth.client.keycloak.AtlasKeycloakClient; +import org.apache.atlas.authorize.AtlasAuthorizationUtils; +import org.apache.atlas.authorize.AtlasEntityAccessRequest; +import org.apache.atlas.authorize.AtlasPrivilege; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.model.instance.AtlasObjectId; +import org.apache.atlas.model.instance.AtlasRelatedObjectId; +import org.apache.atlas.model.instance.AtlasRelationship; +import org.apache.atlas.model.instance.AtlasStruct; +import org.apache.atlas.model.instance.EntityMutations; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.AtlasEntityStore; +import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; +import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; +import org.apache.atlas.type.AtlasEntityType; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.keycloak.representations.idm.RoleRepresentation; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; +import static org.apache.atlas.AtlasErrorCode.OPERATION_NOT_SUPPORTED; +import static org.apache.atlas.repository.Constants.NAME; +import static org.apache.atlas.repository.Constants.POLICY_ENTITY_TYPE; +import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_ACCESS_CONTROL_ENABLED; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_PERSONA_GROUPS; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_PERSONA_ROLE_ID; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_PERSONA_USERS; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_IS_ENABLED; +import static org.apache.atlas.repository.util.AccessControlUtils.REL_ATTR_POLICIES; +import static org.apache.atlas.repository.util.AccessControlUtils.getESAliasName; +import static org.apache.atlas.repository.util.AccessControlUtils.getEntityName; +import static org.apache.atlas.repository.util.AccessControlUtils.getIsAccessControlEnabled; +import static org.apache.atlas.repository.util.AccessControlUtils.getPersonaGroups; +import static org.apache.atlas.repository.util.AccessControlUtils.getPersonaRoleId; +import static org.apache.atlas.repository.util.AccessControlUtils.getPersonaRoleName; +import static org.apache.atlas.repository.util.AccessControlUtils.getPersonaUsers; +import static org.apache.atlas.repository.util.AccessControlUtils.getTenantId; +import static org.apache.atlas.repository.util.AccessControlUtils.getUUID; +import static org.apache.atlas.repository.util.AccessControlUtils.validateNoPoliciesAttached; + +public class StakeholderPreProcessor extends PersonaPreProcessor { + private static final Logger LOG = LoggerFactory.getLogger(StakeholderPreProcessor.class); + + public StakeholderPreProcessor(AtlasGraph graph, + AtlasTypeRegistry typeRegistry, + EntityGraphRetriever entityRetriever, + AtlasEntityStore entityStore) { + super(graph, typeRegistry, entityRetriever, entityStore); + } + + @Override + public void processAttributes(AtlasStruct entityStruct, EntityMutationContext context, + EntityMutations.EntityOperation operation) throws AtlasBaseException { + if (LOG.isDebugEnabled()) { + LOG.debug("StakeholderPreProcessor.processAttributes: pre processing {}, {}", entityStruct.getAttribute(QUALIFIED_NAME), operation); + } + + AtlasEntity entity = (AtlasEntity) entityStruct; + + switch (operation) { + case CREATE: + processCreateStakeholder(entity); + break; + case UPDATE: + processUpdateStakeholder(context, entity); + break; + } + } + + @Override + public void processDelete(AtlasVertex vertex) throws AtlasBaseException { + AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = entityRetriever.toAtlasEntityWithExtInfo(vertex); + AtlasEntity persona = entityWithExtInfo.getEntity(); + + if(!persona.getStatus().equals(AtlasEntity.Status.ACTIVE)) { + LOG.info("Persona with guid {} is already deleted/purged", persona.getGuid()); + return; + } + + //delete policies + List policies = (List) persona.getRelationshipAttribute(REL_ATTR_POLICIES); + if (CollectionUtils.isNotEmpty(policies)) { + for (AtlasObjectId policyObjectId : policies) { + //AtlasVertex policyVertex = entityRetriever.getEntityVertex(policyObjectId.getGuid()); + entityStore.deleteById(policyObjectId.getGuid()); + } + } + + //remove role + keycloakStore.removeRole(getPersonaRoleId(persona)); + + //delete ES alias + aliasStore.deleteAlias(getESAliasName(persona)); + } + + private void processCreateStakeholder(AtlasEntity entity) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreateStakeholder"); + + validateNoPoliciesAttached(entity); + + if (!entity.hasRelationshipAttribute("stakeholderTitle") || !entity.hasRelationshipAttribute("dataDomain")) { + throw new AtlasBaseException(BAD_REQUEST, "Relations stakeholderTitle and dataDomain are mandatory"); + } + + String domainQualifiedName = getQualifiedNameFromRelationAttribute(entity, "dataDomain"); + entity.setAttribute("domainQualifiedName", domainQualifiedName); + entity.setAttribute("stakeholderTitleGuid", getGuidFromRelationAttribute(entity, "stakeholderTitle")); + + String personaQualifiedName = String.format("default/%s/%s", + getUUID(), + domainQualifiedName); + + entity.setAttribute(QUALIFIED_NAME, personaQualifiedName); + + //TODO: validate Stakeholder & StakeholderTitle pair is unique + + + + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, new AtlasEntityHeader(entity)), + "create Stakeholder: ", entity.getAttribute(NAME)); + + entity.setAttribute(ATTR_ACCESS_CONTROL_ENABLED, entity.getAttributes().getOrDefault(ATTR_ACCESS_CONTROL_ENABLED, true)); + + //create keycloak role + String roleId = createKeycloakRole(entity); + + entity.setAttribute(ATTR_PERSONA_ROLE_ID, roleId); + + //create ES alias + aliasStore.createAlias(entity); + + RequestContext.get().endMetricRecord(metricRecorder); + } + + private void processUpdateStakeholder(EntityMutationContext context, AtlasEntity stakeholder) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdateStakeholder"); + + validateNoPoliciesAttached(stakeholder); + + validateNoPoliciesAttached(stakeholder); + AtlasVertex vertex = context.getVertex(stakeholder.getGuid()); + + AtlasEntity existingStakeholderEntity = entityRetriever.toAtlasEntityWithExtInfo(vertex).getEntity(); + + if (!AtlasEntity.Status.ACTIVE.equals(existingStakeholderEntity.getStatus())) { + throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Stakeholder not Active"); + } + + String domainGuid = (String) stakeholder.getAttribute("domainGuid"); + if (StringUtils.isNotEmpty(domainGuid)) { + stakeholder.removeAttribute("domainGuid"); + stakeholder.removeAttribute("stakeholderTitleGuid"); + stakeholder.getRelationshipAttributes().remove("dataDomain"); + stakeholder.getRelationshipAttributes().remove("stakeholderTitle"); + } + + String vertexQName = vertex.getProperty(QUALIFIED_NAME, String.class); + stakeholder.setAttribute(QUALIFIED_NAME, vertexQName); + stakeholder.setAttribute(ATTR_PERSONA_ROLE_ID, getPersonaRoleId(existingStakeholderEntity)); + + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, new AtlasEntityHeader(stakeholder)), + "update Stakeholder: ", stakeholder.getAttribute(NAME)); + + updateKeycloakRole(stakeholder, existingStakeholderEntity); + + RequestContext.get().endMetricRecord(metricRecorder); + } + + private String getGuidFromRelationAttribute(AtlasEntity entity, String relationshipAttributeName) throws AtlasBaseException { + AtlasObjectId relationObjectId = (AtlasObjectId) entity.getRelationshipAttribute(relationshipAttributeName); + + String guid = relationObjectId.getGuid(); + if (StringUtils.isEmpty(guid)) { + AtlasVertex vertex = entityRetriever.getEntityVertex(relationObjectId); + guid = vertex.getProperty("__guid", String.class); + } + + return guid; + } + + private String getQualifiedNameFromRelationAttribute(AtlasEntity entity, String relationshipAttributeName) throws AtlasBaseException { + AtlasObjectId relationObjectId = (AtlasObjectId) entity.getRelationshipAttribute(relationshipAttributeName); + String qualifiedName = null; + + if (relationObjectId.getUniqueAttributes() != null) { + qualifiedName = (String) relationObjectId.getUniqueAttributes().get(QUALIFIED_NAME); + } + + if (StringUtils.isEmpty(qualifiedName)) { + AtlasVertex vertex = entityRetriever.getEntityVertex(relationObjectId); + qualifiedName = vertex.getProperty(QUALIFIED_NAME, String.class); + } + + return qualifiedName; + } +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java index b30f6b3c3a6..52731c64311 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java @@ -104,6 +104,10 @@ private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Cannot update Domain's subDomains or dataProducts relations"); } + if(entity.hasRelationshipAttribute("stakeholders")){ + throw new AtlasBaseException(AtlasErrorCode.OPERATION_NOT_SUPPORTED, "Managing Stakeholders via Domain update"); + } + String vertexQnName = vertex.getProperty(QUALIFIED_NAME, String.class); AtlasEntity storedDomain = entityRetriever.toAtlasEntity(vertex); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java index b35840af133..86bab952e78 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java @@ -78,7 +78,7 @@ private void processCreateStakeholderTitle(AtlasEntity entity) throws AtlasBaseE try { if (entity.hasRelationshipAttribute("stakeholders")) { - throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Can not attach a Stakeholder while creating StakeholderTitle"); + throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Managing Stakeholders while creating StakeholderTitle"); } if (RequestContext.get().isSkipAuthorizationCheck()) { @@ -120,7 +120,7 @@ private void processUpdateStakeholderTitle(EntityMutationContext context, AtlasE } if (entity.hasRelationshipAttribute("stakeholders")) { - throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Can not attach/detach a Stakeholder while updating StakeholderTitle"); + throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Managing Stakeholders while updating StakeholderTitle"); } AtlasVertex vertex = context.getVertex(entity.getGuid()); diff --git a/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java b/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java index dd2a035c6b5..4ebf8c8a661 100644 --- a/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java @@ -337,7 +337,7 @@ public static String getTenantId(AtlasStruct entity) { public static void validateNoPoliciesAttached(AtlasEntity entity) throws AtlasBaseException { List policies = (List) entity.getRelationshipAttribute(REL_ATTR_POLICIES); if (CollectionUtils.isNotEmpty(policies)) { - throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Can not attach a policy while creating/updating Persona/Purpose"); + throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Can not attach a policy while creating/updating Persona/Purpose/Stakeholder"); } } From ef3941fdfa8884492d280a5f3b8b1c393f0928d0 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Thu, 9 May 2024 20:35:11 +0530 Subject: [PATCH 151/316] DG-1235 Skip initial auth checks --- .../main/java/org/apache/atlas/repository/Constants.java | 8 +++++--- .../store/graph/v2/preprocessor/PreProcessor.java | 6 ++++-- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index c8c68b87faf..620799dace4 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -158,7 +158,9 @@ public final class Constants { public static final String PURPOSE_ENTITY_TYPE = "Purpose"; public static final String POLICY_ENTITY_TYPE = "AuthPolicy"; public static final String SERVICE_ENTITY_TYPE = "AuthService"; - public static final String STAKEHOLDER_ENTITY_TYPE = "Stakeholder"; + + public static final String STAKEHOLDER_ENTITY_TYPE = "Stakeholder"; + public static final String STAKEHOLDER_TITLE_ENTITY_TYPE = "StakeholderTitle"; /** * Resource @@ -432,8 +434,8 @@ public enum SupportedFileExtensions { XLSX, XLS, CSV } public static final Set SKIP_UPDATE_AUTH_CHECK_TYPES = new HashSet() {{ add(README_ENTITY_TYPE); add(LINK_ENTITY_TYPE); - add("Persona"); - add("StakeholderTitle"); + add(STAKEHOLDER_ENTITY_TYPE); + add(STAKEHOLDER_TITLE_ENTITY_TYPE); }}; public static final Set SKIP_DELETE_AUTH_CHECK_TYPES = new HashSet() {{ diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessor.java index 79119408f0c..45486e69176 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessor.java @@ -11,6 +11,8 @@ import static org.apache.atlas.repository.Constants.ATLAS_GLOSSARY_CATEGORY_ENTITY_TYPE; import static org.apache.atlas.repository.Constants.ATLAS_GLOSSARY_TERM_ENTITY_TYPE; +import static org.apache.atlas.repository.Constants.STAKEHOLDER_ENTITY_TYPE; +import static org.apache.atlas.repository.Constants.STAKEHOLDER_TITLE_ENTITY_TYPE; public interface PreProcessor { @@ -18,8 +20,8 @@ public interface PreProcessor { Set skipInitialAuthCheckTypes = new HashSet() {{ add(ATLAS_GLOSSARY_TERM_ENTITY_TYPE); add(ATLAS_GLOSSARY_CATEGORY_ENTITY_TYPE); - add("Persona"); - add("StakeholderTitle"); + add(STAKEHOLDER_ENTITY_TYPE); + add(STAKEHOLDER_TITLE_ENTITY_TYPE); }}; void processAttributes(AtlasStruct entity, EntityMutationContext context, EntityMutations.EntityOperation operation) throws AtlasBaseException; From 424df19846539cef2db25f51d8934b98bea2c1e5 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Thu, 9 May 2024 21:03:22 +0530 Subject: [PATCH 152/316] DG-1235 save domain qualified names for Stakeholder title --- .../StakeholderTitlePreProcessor.java | 49 +++++++++++++------ 1 file changed, 35 insertions(+), 14 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java index 86bab952e78..1fe3e35e9ff 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java @@ -27,13 +27,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.Collections; import java.util.List; import java.util.Optional; +import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; import static org.apache.atlas.AtlasErrorCode.OPERATION_NOT_SUPPORTED; +import static org.apache.atlas.repository.Constants.DATA_DOMAIN_ENTITY_TYPE; import static org.apache.atlas.repository.Constants.NAME; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.getUUID; +import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; public class StakeholderTitlePreProcessor implements PreProcessor { @@ -85,26 +89,43 @@ private void processCreateStakeholderTitle(AtlasEntity entity) throws AtlasBaseE return; } - String qualifiedName; + List domainQualifiedNames = null; + + if (entity.hasAttribute("domainQualifiedNames")) { + domainQualifiedNames = (List) entity.getAttribute("domainQualifiedNames"); + if (domainQualifiedNames.size() == 0) { + throw new AtlasBaseException(BAD_REQUEST, "Please pass attribute domainQualifiedNames"); + } - String domainGuid = (String) entity.getAttribute("domainGuid"); - if ("*".equals(domainGuid)) { - qualifiedName = String.format("stakeholderTitle/domain/default/%s", getUUID()); - //TODO: validate name duplication } else { + throw new AtlasBaseException(BAD_REQUEST, "Please pass attribute domainQualifiedNames"); + } - AtlasVertex domain = entityRetriever.getEntityVertex(domainGuid); - qualifiedName = String.format("stakeholderTitle/domain/%s/%s", - getUUID(), - domain.getProperty(QUALIFIED_NAME, String.class)); + if ((domainQualifiedNames.size() == 1 && "*".equals(domainQualifiedNames.get(0))) + || domainQualifiedNames.contains("*")) { - //TODO: validate name duplication - } + AtlasEntityHeader allDomainEntityHeader = new AtlasEntityHeader(DATA_DOMAIN_ENTITY_TYPE, mapOf(QUALIFIED_NAME, "*/super")); + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, new AtlasEntityHeader(allDomainEntityHeader)), + "create StakeholderTitle for all domains"); - entity.setAttribute(QUALIFIED_NAME, qualifiedName); + String qualifiedName = String.format("stakeholderTitle/domain/default/%s", getUUID()); + entity.setAttribute(QUALIFIED_NAME, qualifiedName); + entity.setAttribute("domainQualifiedNames", Collections.singletonList("*")); + } else { + for (String domainQualifiedName : domainQualifiedNames) { + //AtlasVertex domainVertex = entityRetriever.getEntityVertex(new AtlasObjectId(DATA_DOMAIN_ENTITY_TYPE, mapOf(QUALIFIED_NAME, domainQualifiedName))); + //AtlasEntityHeader domainHeader = entityRetriever.toAtlasEntityHeader(domainVertex); + AtlasEntityHeader domainHeader = new AtlasEntityHeader(DATA_DOMAIN_ENTITY_TYPE, mapOf(QUALIFIED_NAME, domainQualifiedName)); + String qualifiedName = String.format("stakeholderTitle/domain/%s/%s", getUUID(), domainQualifiedName); - AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, new AtlasEntityHeader(entity)), - "create StakeholderTitle: ", entity.getAttribute(NAME)); + entity.setAttribute(QUALIFIED_NAME, qualifiedName); + + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, new AtlasEntityHeader(domainHeader)), + "create StakeholderTitle for domain ", domainQualifiedName); + } + + entity.setAttribute(QUALIFIED_NAME, String.format("stakeholderTitle/domain/%s", getUUID())); + } } finally { RequestContext.get().endMetricRecord(metricRecorder); From 2566c2c16149608310de807b978ece33044b0bbc Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Thu, 9 May 2024 23:54:13 +0530 Subject: [PATCH 153/316] Fixed bootstrap default StakeholderTitles --- addons/policies/global_stakeholder-titles.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/addons/policies/global_stakeholder-titles.json b/addons/policies/global_stakeholder-titles.json index 65b2e098be3..a0c3cf6b0a8 100644 --- a/addons/policies/global_stakeholder-titles.json +++ b/addons/policies/global_stakeholder-titles.json @@ -6,7 +6,7 @@ { "qualifiedName": "stakeholderTitle/default/DOMAIN_OWNERS", "name": "Domain Owners", - "domainGuid": "*" + "domainQualifiedNames": ["*"] } }, { @@ -15,7 +15,7 @@ { "qualifiedName": "stakeholderTitle/default/DATA_STEWARDS", "name": "Data Stewards", - "domainGuid": "*" + "domainQualifiedNames": ["*"] } } ] From 9c35a27f9fa8ff54ab876fa686997c06f612f059 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Fri, 10 May 2024 00:14:35 +0530 Subject: [PATCH 154/316] Fixed StakeholderTitles to Stakeholer relationship permission --- .../policy_cache_transformer_persona.json | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/addons/static/templates/policy_cache_transformer_persona.json b/addons/static/templates/policy_cache_transformer_persona.json index 2ae85d577d8..fdcf4e90249 100644 --- a/addons/static/templates/policy_cache_transformer_persona.json +++ b/addons/static/templates/policy_cache_transformer_persona.json @@ -437,6 +437,24 @@ ], "actions": ["add-relationship", "update-relationship", "remove-relationship"] }, + { + "policyResourceCategory": "RELATIONSHIP", + "policyType": "ACCESS", + "description": "Link/unlink any Stakeholder Title to this Domain's Stakeholder", + + "resources": [ + "relationship-type:*", + + "end-one-entity-type:StakeholderTitle", + "end-one-entity-classification:*", + "end-one-entity:*", + + "end-two-entity-type:Stakeholder", + "end-two-entity-classification:*", + "end-two-entity:default/*/{entity}" + ], + "actions": ["add-relationship", "update-relationship", "remove-relationship"] + }, { "policyResourceCategory": "ENTITY", "policyType": "ACCESS", @@ -625,6 +643,24 @@ ], "actions": ["add-relationship", "update-relationship", "remove-relationship"] }, + { + "policyResourceCategory": "RELATIONSHIP", + "policyType": "ACCESS", + "description": "Link/unlink any Stakeholder Title to sub-domains's Stakeholder", + + "resources": [ + "relationship-type:*", + + "end-one-entity-type:StakeholderTitle", + "end-one-entity-classification:*", + "end-one-entity:*", + + "end-two-entity-type:Stakeholder", + "end-two-entity-classification:*", + "end-two-entity:default/*/{entity}/*" + ], + "actions": ["add-relationship", "update-relationship", "remove-relationship"] + }, { "policyResourceCategory": "ENTITY", "policyType": "ACCESS", From 7dd290d0b8d23ed70a72ada1626f78b1b9e29eff Mon Sep 17 00:00:00 2001 From: hr2904 Date: Fri, 10 May 2024 13:28:09 +0530 Subject: [PATCH 155/316] Single commit regarding the migration api for QN of datamesh entities. --- .../atlas/repository/graph/GraphHelper.java | 23 + .../v2/DataDomainQNMigrationService.java | 408 ++++++++++++++++++ .../store/graph/v2/MigrationService.java | 5 + .../v2/preprocessor/PreProcessorUtils.java | 9 + .../apache/atlas/web/rest/MigrationREST.java | 67 ++- 5 files changed, 511 insertions(+), 1 deletion(-) create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataDomainQNMigrationService.java create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/MigrationService.java diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index 8844d2d87bc..0d403c981f2 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -1924,6 +1924,29 @@ public static Iterator getActiveVertices(AtlasVertex vertex, String } } + public static Iterator getAllChildrenVertices(AtlasVertex vertex, String childrenEdgeLabel) throws AtlasBaseException { + return getAllVertices(vertex, childrenEdgeLabel, AtlasEdgeDirection.OUT); + } + + public static Iterator getAllVertices(AtlasVertex vertex, String childrenEdgeLabel, AtlasEdgeDirection direction) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("CategoryPreProcessor.getEdges"); + + try { + return vertex.query() + .direction(direction) + .label(childrenEdgeLabel) + .vertices() + .iterator(); + } catch (Exception e) { + LOG.error("Error while getting all children of category for edge label " + childrenEdgeLabel, e); + throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, e); + } + finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + } + + private static Set parseLabelsString(String labels) { Set ret = new HashSet<>(); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataDomainQNMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataDomainQNMigrationService.java new file mode 100644 index 00000000000..c11f38651ba --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataDomainQNMigrationService.java @@ -0,0 +1,408 @@ +package org.apache.atlas.repository.store.graph.v2; + +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.RequestContext; +import org.apache.atlas.discovery.EntityDiscoveryService; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.discovery.IndexSearchParams; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.repository.graph.GraphHelper; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.AtlasEntityStore; +import org.apache.atlas.service.redis.RedisService; +import org.apache.atlas.type.AtlasEntityType; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.util.NanoIdUtils; +import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.lang.StringUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.scheduling.annotation.Async; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import javax.inject.Named; +import java.util.*; + +import static org.apache.atlas.repository.Constants.*; +import static org.apache.atlas.repository.Constants.POLICY_ENTITY_TYPE; +import static org.apache.atlas.repository.graph.GraphHelper.getAllChildrenVertices; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.*; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_CATEGORY; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_RESOURCES; + +@Component +public class DataDomainQNMigrationService implements MigrationService{ + + private static final Logger LOG = LoggerFactory.getLogger(DataDomainQNMigrationService.class); + + private final AtlasEntityStore entityStore; + private final EntityDiscoveryService discovery; + private final EntityGraphRetriever entityRetriever; + + protected final AtlasTypeRegistry typeRegistry; + private Map updatedPolicyResources; + + private final int BATCH_SIZE = 20; + + boolean errorOccur = false; + + boolean skipSuperDomain = false; + + private int Counter; + + private final TransactionInterceptHelper transactionInterceptHelper; + private final RedisService redisService; + + @Inject + public DataDomainQNMigrationService(AtlasEntityStore entityStore, RedisService redisService, EntityDiscoveryService discovery, EntityGraphRetriever entityRetriever, AtlasTypeRegistry typeRegistry, TransactionInterceptHelper transactionInterceptHelper) { + this.entityRetriever = entityRetriever; + this.entityStore = entityStore; + this.discovery = discovery; + this.typeRegistry = typeRegistry; + this.updatedPolicyResources = new HashMap<>(); + this.Counter = 0; + this.transactionInterceptHelper = transactionInterceptHelper; + this.redisService = redisService; + } + + @Async + public Boolean startMigration () throws Exception{ + redisService.putValue(DATA_MESH_QN,IN_PROGRESS); + Set attributes = new HashSet<>(Arrays.asList(QUALIFIED_NAME, SUPER_DOMAIN_QN_ATTR, PARENT_DOMAIN_QN_ATTR, "__customAttributes")); + List entities = null; + entities = getEntity(DATA_DOMAIN_ENTITY_TYPE, attributes, null); + + + for (int i = 0; i < entities.size(); i++) { + skipSuperDomain = false; + updateChunk(entities.get(i)); + } + if(errorOccur) + redisService.putValue(DATA_MESH_QN,FAILED); + else + redisService.putValue(DATA_MESH_QN,SUCCESSFUL); + LOG.info("MIGRATION_RESULT",redisService.getValue("MIGRATION_DATA_DOMAIN_QN")); + return Boolean.TRUE; + } + + private void updateChunk (AtlasEntityHeader atlasEntity) throws AtlasBaseException { + AtlasVertex vertex = entityRetriever.getEntityVertex(atlasEntity.getGuid()); + String qualifiedName = (String) atlasEntity.getAttribute(QUALIFIED_NAME); + + try{ + migrateDomainAttributes(vertex, "", ""); + } + catch (AtlasBaseException e){ + this.errorOccur = true; + LOG.error("Error while migrating qualified name for entity: {}", qualifiedName, e); + } + finally { + commitChanges(); + LOG.info("Migrated qualified name for entity: {}", qualifiedName); + } + } + + + + private void migrateDomainAttributes(AtlasVertex vertex, String parentDomainQualifiedName, String rootDomainQualifiedName) throws AtlasBaseException { + if(skipSuperDomain) + return; + String currentQualifiedName = vertex.getProperty(QUALIFIED_NAME,String.class); + LOG.info("Migrating qualified name for Domain: {}", currentQualifiedName); + Counter++; + Map updatedAttributes = new HashMap<>(); + String updatedQualifiedName = createDomainQualifiedName(parentDomainQualifiedName); + Map customAttributes = GraphHelper.getCustomAttributes(vertex); + if(customAttributes != null && customAttributes.get(MIGRATION_CUSTOM_ATTRIBUTE) != null && customAttributes.get(MIGRATION_CUSTOM_ATTRIBUTE).equals("true")){ + LOG.info("Entity already migrated for entity: {}", currentQualifiedName); + } + else{ + rootDomainQualifiedName = commitChangesInMemory(currentQualifiedName,updatedQualifiedName,parentDomainQualifiedName,rootDomainQualifiedName,vertex,updatedAttributes); + } + if(!skipSuperDomain) { + Iterator products = getAllChildrenVertices(vertex, DATA_PRODUCT_EDGE_LABEL); + + while (products.hasNext()) { + AtlasVertex productVertex = products.next(); + migrateDataProductAttributes(productVertex, updatedQualifiedName, rootDomainQualifiedName); + if(skipSuperDomain) + break; + } + + // Get all children domains of current domain + Iterator childDomains = getAllChildrenVertices(vertex, DOMAIN_PARENT_EDGE_LABEL); + + while (childDomains.hasNext()) { + AtlasVertex childVertex = childDomains.next(); + migrateDomainAttributes(childVertex, updatedQualifiedName, rootDomainQualifiedName); + if(skipSuperDomain) + break; + } + + recordUpdatedChildEntities(vertex, updatedAttributes); + if (Counter >= BATCH_SIZE) { + commitChanges(); + } + } + + } + + public void commitChanges(){ + try { + updatePolicy(this.updatedPolicyResources); + } catch (AtlasBaseException e) { + this.errorOccur = true; + throw new RuntimeException(e); + } + try { + transactionInterceptHelper.intercept(); + }catch (Exception e){ + this.skipSuperDomain = true; + this.errorOccur = true; + } + this.updatedPolicyResources.clear(); + this.Counter = 0; + } + public String commitChangesInMemory(String currentQualifiedName, String updatedQualifiedName, String parentDomainQualifiedName, String rootDomainQualifiedName, AtlasVertex vertex, Map updatedAttributes){ + if(skipSuperDomain) + return ""; + vertex.setProperty(QUALIFIED_NAME, updatedQualifiedName); + if (StringUtils.isEmpty(parentDomainQualifiedName) && StringUtils.isEmpty(rootDomainQualifiedName)){ + rootDomainQualifiedName = updatedQualifiedName; + } + else{ + vertex.setProperty(PARENT_DOMAIN_QN_ATTR, parentDomainQualifiedName); + vertex.setProperty(SUPER_DOMAIN_QN_ATTR, rootDomainQualifiedName); + } + + updatedAttributes.put(QUALIFIED_NAME, updatedQualifiedName); + + //Store domainPolicies and resources to be updated + String currentResource = "entity:"+ currentQualifiedName; + String updatedResource = "entity:"+ updatedQualifiedName; + this.updatedPolicyResources.put(currentResource, updatedResource); + + Map customAttributes = new HashMap<>(); + customAttributes.put(MIGRATION_CUSTOM_ATTRIBUTE, "true"); + vertex.setProperty(CUSTOM_ATTRIBUTES_PROPERTY_KEY, AtlasEntityType.toJson(customAttributes)); + + return rootDomainQualifiedName; + } + + + private void migrateDataProductAttributes(AtlasVertex vertex, String parentDomainQualifiedName, String rootDomainQualifiedName) throws AtlasBaseException { + if(skipSuperDomain) + return; + Counter++; + String currentQualifiedName = (String) vertex.getProperty(QUALIFIED_NAME,String.class); + LOG.info("Migrating qualified name for Product: {}", currentQualifiedName); + Map customAttributes_ = GraphHelper.getCustomAttributes(vertex); + String updatedQualifiedName = createProductQualifiedName(parentDomainQualifiedName); + if(customAttributes_ != null && customAttributes_.get(MIGRATION_CUSTOM_ATTRIBUTE) != null && customAttributes_.get(MIGRATION_CUSTOM_ATTRIBUTE).equals("true")){ + LOG.info("Entity already migrated for entity: {}", currentQualifiedName); + } + else { + vertex.setProperty(QUALIFIED_NAME, updatedQualifiedName); + + //Store domainPolicies and resources to be updated + String currentResource = "entity:" + currentQualifiedName; + String updatedResource = "entity:" + updatedQualifiedName; + this.updatedPolicyResources.put(currentResource, updatedResource); + + vertex.setProperty(PARENT_DOMAIN_QN_ATTR, parentDomainQualifiedName); + vertex.setProperty(SUPER_DOMAIN_QN_ATTR, rootDomainQualifiedName); + + Map customAttributes = new HashMap<>(); + customAttributes.put(MIGRATION_CUSTOM_ATTRIBUTE, "true"); + vertex.setProperty(CUSTOM_ATTRIBUTES_PROPERTY_KEY, AtlasEntityType.toJson(customAttributes)); + } + if(Counter >= BATCH_SIZE){ + commitChanges(); + } + } + + protected void updatePolicy(Map updatedPolicyResources) throws AtlasBaseException { + if(skipSuperDomain) + return; + List currentResources = new ArrayList<>(updatedPolicyResources.keySet()); + LOG.info("Updating policies for entities {}", currentResources); + Map updatedAttributes = new HashMap<>(); + + List policies = getEntity(POLICY_ENTITY_TYPE,new HashSet<>(Arrays.asList(ATTR_POLICY_RESOURCES, ATTR_POLICY_CATEGORY)), currentResources); + if (CollectionUtils.isNotEmpty(policies)) { + int batchSize = BATCH_SIZE; + int totalPolicies = policies.size(); + + for (int i = 0; i < totalPolicies; i += batchSize) { + List entityList = new ArrayList<>(); + List batch = policies.subList(i, Math.min(i + batchSize, totalPolicies)); + + for (AtlasEntityHeader policy : batch) { + AtlasVertex policyVertex = entityRetriever.getEntityVertex(policy.getGuid()); + AtlasEntity policyEntity = entityRetriever.toAtlasEntity(policyVertex); + + List policyResources = (List) policyEntity.getAttribute(ATTR_POLICY_RESOURCES); + List updatedPolicyResourcesList = new ArrayList<>(); + + for (String resource : policyResources) { + if (updatedPolicyResources.containsKey(resource)) { + updatedPolicyResourcesList.add(updatedPolicyResources.get(resource)); + } else { + updatedPolicyResourcesList.add(resource); + } + } + updatedAttributes.put(ATTR_POLICY_RESOURCES, updatedPolicyResourcesList); + + policyEntity.setAttribute(ATTR_POLICY_RESOURCES, updatedPolicyResourcesList); + entityList.add(policyEntity); + recordUpdatedChildEntities(policyVertex, updatedAttributes); + } + + EntityStream entityStream = new AtlasEntityStream(entityList); + entityStore.createOrUpdate(entityStream, false); + } + + } + + } + + private static String createDomainQualifiedName(String parentDomainQualifiedName) { + if (StringUtils.isNotEmpty(parentDomainQualifiedName)) { + return parentDomainQualifiedName + "/domain/" + getUUID(); + } else{ + return "default/domain" + "/" + getUUID() + "/super"; + } + } + + private static String createProductQualifiedName(String parentDomainQualifiedName) throws AtlasBaseException { + if (StringUtils.isEmpty(parentDomainQualifiedName)) { + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Parent Domain Qualified Name cannot be empty or null"); + } + return parentDomainQualifiedName + "/product/" + getUUID(); + } + + public static String getUUID(){ + return NanoIdUtils.randomNanoId(); + } + + public List getEntity(String entityType, Set attributes, List resource) throws AtlasBaseException { + + List> mustClauseList = new ArrayList<>(); + mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", entityType))); + + if(entityType.equals(DATA_DOMAIN_ENTITY_TYPE)){ + Map childBool = new HashMap<>(); + List > mustNotClauseList = new ArrayList<>(); + mustNotClauseList.add(mapOf("exists", mapOf("field", PARENT_DOMAIN_QN_ATTR))); + + Map shouldBool = new HashMap<>(); + shouldBool.put("must_not", mustNotClauseList); + + List > shouldClauseList = new ArrayList<>(); + shouldClauseList.add(mapOf("bool", shouldBool)); + + childBool.put("should", shouldClauseList); + mustClauseList.add(mapOf("bool", childBool)); + } + + if(entityType.equals(POLICY_ENTITY_TYPE)){ + mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); + mustClauseList.add(mapOf("terms", mapOf("policyResources", resource))); + } + + Map bool = new HashMap<>(); + bool.put("must", mustClauseList); + + Map dsl = mapOf("query", mapOf("bool", bool)); + + List> sortList = new ArrayList<>(); + Map sortField = new HashMap<>(); + sortField.put("__timestamp", mapOf("order", "DESC")); + sortList.add(sortField); + dsl.put("sort", sortList); + + + List entities = indexSearchPaginated(dsl, attributes, discovery); + + return entities; + } + + public static List indexSearchPaginated(Map dsl, Set attributes, EntityDiscoveryService discovery) throws AtlasBaseException { + IndexSearchParams searchParams = new IndexSearchParams(); + List ret = new ArrayList<>(); + + List sortList = new ArrayList<>(0); + sortList.add(mapOf("__timestamp", mapOf("order", "asc"))); + sortList.add(mapOf("__guid", mapOf("order", "asc"))); + dsl.put("sort", sortList); + + int from = 0; + int size = 100; + boolean hasMore = true; + do { + dsl.put("from", from); + dsl.put("size", size); + searchParams.setDsl(dsl); + + if (CollectionUtils.isNotEmpty(attributes)) { + searchParams.setAttributes(attributes); + } + + List headers = discovery.directIndexSearch(searchParams).getEntities(); + + if (CollectionUtils.isNotEmpty(headers)) { + ret.addAll(headers); + } else { + hasMore = false; + } + + from += size; + + } while (hasMore); + + return ret; + } + + /** + * Record the updated child entities, it will be used to send notification and store audit logs + * @param entityVertex Child entity vertex + * @param updatedAttributes Updated attributes while updating required attributes on updating collection + */ + protected void recordUpdatedChildEntities(AtlasVertex entityVertex, Map updatedAttributes) { + RequestContext requestContext = RequestContext.get(); + + AtlasEntity entity = new AtlasEntity(); + entity = entityRetriever.mapSystemAttributes(entityVertex, entity); + entity.setAttributes(updatedAttributes); + requestContext.cacheDifferentialEntity(new AtlasEntity(entity)); + + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(entity.getTypeName()); + + //Add the min info attributes to entity header to be sent as part of notification + if(entityType != null) { + AtlasEntity finalEntity = entity; + entityType.getMinInfoAttributes().values().stream().filter(attribute -> !updatedAttributes.containsKey(attribute.getName())).forEach(attribute -> { + Object attrValue = null; + try { + attrValue = entityRetriever.getVertexAttribute(entityVertex, attribute); + } catch (AtlasBaseException e) { + this.errorOccur = true; + LOG.error("Error while getting vertex attribute", e); + } + if(attrValue != null) { + finalEntity.setAttribute(attribute.getName(), attrValue); + } + }); + requestContext.recordEntityUpdate(new AtlasEntityHeader(finalEntity)); + } + + } + + public static Map mapOf(String key, Object value) { + Map map = new HashMap<>(); + map.put(key, value); + return map; + } +} \ No newline at end of file diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/MigrationService.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/MigrationService.java new file mode 100644 index 00000000000..91d80c65540 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/MigrationService.java @@ -0,0 +1,5 @@ +package org.apache.atlas.repository.store.graph.v2; + +public interface MigrationService { + Boolean startMigration() throws Exception; +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index c8728df251c..1de077c3e50 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -43,6 +43,7 @@ public class PreProcessorUtils { public static final String PARENT_DOMAIN_REL_TYPE = "parentDomain"; public static final String SUB_DOMAIN_REL_TYPE = "subDomains"; public static final String DATA_PRODUCT_REL_TYPE = "dataProducts"; + public static final String MIGRATION_CUSTOM_ATTRIBUTE = "isQualifiedNameMigrated"; public static final String DATA_DOMAIN_REL_TYPE = "dataDomain"; public static final String MESH_POLICY_CATEGORY = "datamesh"; @@ -53,6 +54,14 @@ public class PreProcessorUtils { public static final String PARENT_DOMAIN_QN_ATTR = "parentDomainQualifiedName"; public static final String SUPER_DOMAIN_QN_ATTR = "superDomainQualifiedName"; + //Migration Constants + public static final String MIGRATION = "MIGRATION:"; + public static final String DATA_MESH_QN = MIGRATION + "DATA_MESH_QN"; + public static final String IN_PROGRESS = "IN_PROGRESS"; + public static final String SUCCESSFUL = "SUCCESSFUL"; + public static final String PARTIAL_SUCCESSFUL = "PARTIAL_SUCCESSFUL"; + + public static final String FAILED = "FAILED"; //Query models constants public static final String PREFIX_QUERY_QN = "default/collection/"; public static final String COLLECTION_QUALIFIED_NAME = "collectionQualifiedName"; diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java index b491bb88ced..40580325681 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java @@ -12,7 +12,9 @@ import org.apache.atlas.repository.store.graph.AtlasEntityStore; import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream; import org.apache.atlas.repository.store.graph.v2.EntityStream; +import org.apache.atlas.repository.store.graph.v2.MigrationService; import org.apache.atlas.repository.store.users.KeycloakStore; +import org.apache.atlas.service.redis.RedisService; import org.apache.atlas.transformer.PreProcessorPoliciesTransformer; import org.apache.atlas.utils.AtlasPerfTracer; import org.apache.atlas.v1.model.instance.Id; @@ -35,6 +37,8 @@ import static org.apache.atlas.auth.client.keycloak.AtlasKeycloakClient.getKeycloakClient; import static org.apache.atlas.repository.Constants.*; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.DATA_MESH_QN; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.MIGRATION; @Path("migration") @Singleton @@ -53,13 +57,74 @@ public class MigrationREST { private final PreProcessorPoliciesTransformer transformer; private KeycloakStore keycloakStore; private AtlasGraph graph; + private final Map migrationServicesMap = new HashMap<>(); + List migrationServices; + private final RedisService redisService; @Inject - public MigrationREST(AtlasEntityStore entityStore, AtlasGraph graph) { + public MigrationREST(AtlasEntityStore entityStore, AtlasGraph graph, List migrationServices,RedisService redisService) { this.entityStore = entityStore; this.graph = graph; this.transformer = new PreProcessorPoliciesTransformer(); + this.migrationServices = migrationServices; keycloakStore = new KeycloakStore(); + this.redisService = redisService; + for (MigrationService service : migrationServices) { + String[] path = service.getClass().getName().split("\\."); + migrationServicesMap.put(path[path.length - 1], service); + } + } + + @POST + @Path("updateQn") + @Timed + public Boolean updateQn (@QueryParam("migrationType") String migrationType) throws Exception { + AtlasPerfTracer perf = null; + MigrationService migrationService; + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "MigrationREST.updateQn()"); + } + migrationService = getMigrationService(MIGRATION + migrationType); + return Objects.nonNull(migrationService)?migrationService.startMigration():Boolean.FALSE; + } catch (Exception e) { + LOG.error("Error while updating qualified names", e); + throw e; + } finally { + AtlasPerfTracer.log(perf); + } + + } + + private MigrationService getMigrationService(String migrationType){ + switch (migrationType){ + case DATA_MESH_QN: + return migrationServicesMap.get("DataDomainQNMigrationService"); + default: + LOG.info("No service type found"); + return null; + } + } + + @GET + @Path("migrationStatus") + @Timed + public String migrationStatus(@QueryParam("migrationType") String migrationType) throws Exception{ + AtlasPerfTracer perf = null; + MigrationService migrationService = null; + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "MigrationREST.migrationStatus()"); + } + + String value = redisService.getValue(MIGRATION + migrationType); + return Objects.nonNull(value)?value:"No Migration Found with this key"; + } catch (Exception e) { + LOG.error("Error while updating qualified names", e); + throw e; + } finally { + AtlasPerfTracer.log(perf); + } } @POST From 8b74a67124ce633964d5527c5982bc025262daa2 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Fri, 10 May 2024 14:02:21 +0530 Subject: [PATCH 156/316] Modified the addition of customAttribute such that, it doesnt override any other pre existing custom attributes. --- .../store/graph/v2/DataDomainQNMigrationService.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataDomainQNMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataDomainQNMigrationService.java index c11f38651ba..7e6a204874a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataDomainQNMigrationService.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataDomainQNMigrationService.java @@ -15,6 +15,7 @@ import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.util.NanoIdUtils; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -183,8 +184,9 @@ public String commitChangesInMemory(String currentQualifiedName, String updatedQ String currentResource = "entity:"+ currentQualifiedName; String updatedResource = "entity:"+ updatedQualifiedName; this.updatedPolicyResources.put(currentResource, updatedResource); - - Map customAttributes = new HashMap<>(); + Map customAttributes = GraphHelper.getCustomAttributes(vertex); + if(Objects.isNull(customAttributes) || MapUtils.isEmpty(customAttributes)) + customAttributes = new HashMap<>(); customAttributes.put(MIGRATION_CUSTOM_ATTRIBUTE, "true"); vertex.setProperty(CUSTOM_ATTRIBUTES_PROPERTY_KEY, AtlasEntityType.toJson(customAttributes)); @@ -214,7 +216,9 @@ private void migrateDataProductAttributes(AtlasVertex vertex, String parentDomai vertex.setProperty(PARENT_DOMAIN_QN_ATTR, parentDomainQualifiedName); vertex.setProperty(SUPER_DOMAIN_QN_ATTR, rootDomainQualifiedName); - Map customAttributes = new HashMap<>(); + Map customAttributes = GraphHelper.getCustomAttributes(vertex); + if(Objects.isNull(customAttributes) || MapUtils.isEmpty(customAttributes)) + customAttributes = new HashMap<>(); customAttributes.put(MIGRATION_CUSTOM_ATTRIBUTE, "true"); vertex.setProperty(CUSTOM_ATTRIBUTES_PROPERTY_KEY, AtlasEntityType.toJson(customAttributes)); } From 1c5068fa11feab9ae19b063a6f33c6dea570950b Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Fri, 10 May 2024 14:43:58 +0530 Subject: [PATCH 157/316] DG-1235 Vaalidation, refactorings --- .../apache/atlas/repository/Constants.java | 12 +- .../graph/v2/AtlasRelationshipStoreV2.java | 10 +- .../v2/preprocessor/PreProcessorUtils.java | 23 +++ .../accesscontrol/PersonaPreProcessor.java | 1 + .../StakeholderPreProcessor.java | 139 ++++++++++++++---- .../StakeholderTitlePreProcessor.java | 64 ++++++-- 6 files changed, 197 insertions(+), 52 deletions(-) diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 620799dace4..b5fd8a85194 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -141,6 +141,15 @@ public final class Constants { public static final String DATA_DOMAIN_ENTITY_TYPE = "DataDomain"; public static final String DATA_PRODUCT_ENTITY_TYPE = "DataProduct"; + public static final String STAKEHOLDER_ENTITY_TYPE = "Stakeholder"; + public static final String STAKEHOLDER_TITLE_ENTITY_TYPE = "StakeholderTitle"; + + public static final String REL_DOMAIN_TO_DOMAINS = "parent_domain_sub_domains"; + public static final String REL_DOMAIN_TO_PRODUCTS = "data_domain_data_products"; + + public static final String REL_DOMAIN_TO_STAKEHOLDERS = "data_domain_stakeholders"; + public static final String REL_STAKEHOLDER_TITLE_TO_STAKEHOLDERS = "stakeholder_title_stakeholders"; + /** * SQL property keys. @@ -159,9 +168,6 @@ public final class Constants { public static final String POLICY_ENTITY_TYPE = "AuthPolicy"; public static final String SERVICE_ENTITY_TYPE = "AuthService"; - public static final String STAKEHOLDER_ENTITY_TYPE = "Stakeholder"; - public static final String STAKEHOLDER_TITLE_ENTITY_TYPE = "StakeholderTitle"; - /** * Resource */ diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java index 5e9d79b28c7..e2453ba1b43 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java @@ -73,6 +73,10 @@ import static org.apache.atlas.repository.Constants.PROVENANCE_TYPE_KEY; import static org.apache.atlas.repository.Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY; import static org.apache.atlas.repository.Constants.RELATIONSHIP_GUID_PROPERTY_KEY; +import static org.apache.atlas.repository.Constants.REL_DOMAIN_TO_DOMAINS; +import static org.apache.atlas.repository.Constants.REL_DOMAIN_TO_PRODUCTS; +import static org.apache.atlas.repository.Constants.REL_DOMAIN_TO_STAKEHOLDERS; +import static org.apache.atlas.repository.Constants.REL_STAKEHOLDER_TITLE_TO_STAKEHOLDERS; import static org.apache.atlas.repository.Constants.VERSION_PROPERTY_KEY; import static org.apache.atlas.repository.graph.GraphHelper.getTypeName; import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.*; @@ -106,8 +110,10 @@ public class AtlasRelationshipStoreV2 implements AtlasRelationshipStore { private static final String ES_DOC_ID_MAP_KEY = "esDocIdMap"; private static Set EXCLUDE_MUTATION_REL_TYPE_NAMES = new HashSet() {{ - add("parent_domain_sub_domains"); - add("data_domain_data_products"); + add(REL_DOMAIN_TO_DOMAINS); + add(REL_DOMAIN_TO_PRODUCTS); + add(REL_DOMAIN_TO_STAKEHOLDERS); + add(REL_STAKEHOLDER_TITLE_TO_STAKEHOLDERS); }}; public enum RelationshipMutation { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index c8728df251c..8dae53146a7 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -1,5 +1,6 @@ package org.apache.atlas.repository.store.graph.v2.preprocessor; +import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.discovery.IndexSearchParams; @@ -21,9 +22,11 @@ import java.util.*; +import static org.apache.atlas.repository.Constants.NAME; import static org.apache.atlas.repository.Constants.QUERY_COLLECTION_ENTITY_TYPE; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY; +import static org.apache.atlas.repository.Constants.STAKEHOLDER_ENTITY_TYPE; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; public class PreProcessorUtils { @@ -164,4 +167,24 @@ public static List indexSearchPaginated(Map d return ret; } + + public static void verifyDuplicateAssetByName(String typeName, String assetName, EntityDiscoveryService discovery, String errorMessage) throws AtlasBaseException { + boolean exists = false; + + List> mustClauseList = new ArrayList(); + mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", typeName))); + mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); + mustClauseList.add(mapOf("term", mapOf("name.keyword", assetName))); + + + Map bool = mapOf("must", mustClauseList); + + Map dsl = mapOf("query", mapOf("bool", bool)); + + List assets = indexSearchPaginated(dsl, null, discovery); + + if (CollectionUtils.isNotEmpty(assets)) { + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, errorMessage); + } + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java index 5c93b521d72..222bca09628 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/PersonaPreProcessor.java @@ -19,6 +19,7 @@ import org.apache.atlas.RequestContext; +import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.auth.client.keycloak.AtlasKeycloakClient; import org.apache.atlas.model.instance.AtlasEntity; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java index b499a274ce8..949e5aa7b8e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java @@ -18,69 +18,77 @@ package org.apache.atlas.repository.store.graph.v2.preprocessor.accesscontrol; +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.AtlasException; import org.apache.atlas.RequestContext; -import org.apache.atlas.auth.client.keycloak.AtlasKeycloakClient; import org.apache.atlas.authorize.AtlasAuthorizationUtils; import org.apache.atlas.authorize.AtlasEntityAccessRequest; import org.apache.atlas.authorize.AtlasPrivilege; +import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasObjectId; -import org.apache.atlas.model.instance.AtlasRelatedObjectId; -import org.apache.atlas.model.instance.AtlasRelationship; import org.apache.atlas.model.instance.AtlasStruct; import org.apache.atlas.model.instance.EntityMutations; +import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.AtlasEntityStore; -import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; -import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; -import org.keycloak.representations.idm.RoleRepresentation; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Collections; -import java.util.HashMap; +import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Optional; import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; import static org.apache.atlas.AtlasErrorCode.OPERATION_NOT_SUPPORTED; import static org.apache.atlas.repository.Constants.NAME; -import static org.apache.atlas.repository.Constants.POLICY_ENTITY_TYPE; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; +import static org.apache.atlas.repository.Constants.STAKEHOLDER_ENTITY_TYPE; +import static org.apache.atlas.repository.Constants.STAKEHOLDER_TITLE_ENTITY_TYPE; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.indexSearchPaginated; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.verifyDuplicateAssetByName; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.datamesh.StakeholderTitlePreProcessor.ATTR_DOMAIN_QUALIFIED_NAMES; import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_ACCESS_CONTROL_ENABLED; -import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_PERSONA_GROUPS; import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_PERSONA_ROLE_ID; -import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_PERSONA_USERS; -import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_IS_ENABLED; import static org.apache.atlas.repository.util.AccessControlUtils.REL_ATTR_POLICIES; import static org.apache.atlas.repository.util.AccessControlUtils.getESAliasName; -import static org.apache.atlas.repository.util.AccessControlUtils.getEntityName; -import static org.apache.atlas.repository.util.AccessControlUtils.getIsAccessControlEnabled; -import static org.apache.atlas.repository.util.AccessControlUtils.getPersonaGroups; import static org.apache.atlas.repository.util.AccessControlUtils.getPersonaRoleId; -import static org.apache.atlas.repository.util.AccessControlUtils.getPersonaRoleName; -import static org.apache.atlas.repository.util.AccessControlUtils.getPersonaUsers; -import static org.apache.atlas.repository.util.AccessControlUtils.getTenantId; import static org.apache.atlas.repository.util.AccessControlUtils.getUUID; import static org.apache.atlas.repository.util.AccessControlUtils.validateNoPoliciesAttached; +import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; public class StakeholderPreProcessor extends PersonaPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(StakeholderPreProcessor.class); + public static final String ATTR_DOMAIN_QUALIFIED_NAME = "domainQualifiedName"; + public static final String ATTR_STAKEHOLDER_TITLE_GUID = "stakeholderTitleGuid"; + + public static final String REL_ATTR_STAKEHOLDER_TITLE = "stakeholderTitle"; + public static final String REL_ATTR_STAKEHOLDER_DOMAIN = "dataDomain"; + + protected EntityDiscoveryService discovery; + public StakeholderPreProcessor(AtlasGraph graph, AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, AtlasEntityStore entityStore) { super(graph, typeRegistry, entityRetriever, entityStore); + + try { + this.discovery = new EntityDiscoveryService(typeRegistry, graph, null, null, null, null); + } catch (AtlasException e) { + e.printStackTrace(); + } } @Override @@ -133,13 +141,25 @@ private void processCreateStakeholder(AtlasEntity entity) throws AtlasBaseExcept validateNoPoliciesAttached(entity); - if (!entity.hasRelationshipAttribute("stakeholderTitle") || !entity.hasRelationshipAttribute("dataDomain")) { + if (!entity.hasRelationshipAttribute(REL_ATTR_STAKEHOLDER_TITLE) || !entity.hasRelationshipAttribute(REL_ATTR_STAKEHOLDER_DOMAIN)) { throw new AtlasBaseException(BAD_REQUEST, "Relations stakeholderTitle and dataDomain are mandatory"); } - String domainQualifiedName = getQualifiedNameFromRelationAttribute(entity, "dataDomain"); - entity.setAttribute("domainQualifiedName", domainQualifiedName); - entity.setAttribute("stakeholderTitleGuid", getGuidFromRelationAttribute(entity, "stakeholderTitle")); + String domainQualifiedName = getQualifiedNameFromRelationAttribute(entity, REL_ATTR_STAKEHOLDER_DOMAIN); + String stakeholderTitleGuid = getGuidFromRelationAttribute(entity, REL_ATTR_STAKEHOLDER_TITLE); + + ensureTitleAvailableForDomain(domainQualifiedName, stakeholderTitleGuid); + + //validate Stakeholder & StakeholderTitle pair is unique for this domain + verifyDuplicateByDomainAndTitle(domainQualifiedName, stakeholderTitleGuid); + + //validate Name uniqueness for Stakeholders across this domain + String name = (String) entity.getAttribute(NAME); + verifyDuplicateAssetByName(STAKEHOLDER_ENTITY_TYPE, name, discovery, + String.format("Stakeholder with name %s already exists for current domain", name)); + + entity.setAttribute(ATTR_DOMAIN_QUALIFIED_NAME, domainQualifiedName); + entity.setAttribute(ATTR_STAKEHOLDER_TITLE_GUID, stakeholderTitleGuid); String personaQualifiedName = String.format("default/%s/%s", getUUID(), @@ -147,9 +167,6 @@ private void processCreateStakeholder(AtlasEntity entity) throws AtlasBaseExcept entity.setAttribute(QUALIFIED_NAME, personaQualifiedName); - //TODO: validate Stakeholder & StakeholderTitle pair is unique - - AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, new AtlasEntityHeader(entity)), "create Stakeholder: ", entity.getAttribute(NAME)); @@ -181,12 +198,18 @@ private void processUpdateStakeholder(EntityMutationContext context, AtlasEntity throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Stakeholder not Active"); } - String domainGuid = (String) stakeholder.getAttribute("domainGuid"); - if (StringUtils.isNotEmpty(domainGuid)) { - stakeholder.removeAttribute("domainGuid"); - stakeholder.removeAttribute("stakeholderTitleGuid"); - stakeholder.getRelationshipAttributes().remove("dataDomain"); - stakeholder.getRelationshipAttributes().remove("stakeholderTitle"); + stakeholder.removeAttribute(ATTR_DOMAIN_QUALIFIED_NAME); + stakeholder.removeAttribute(ATTR_STAKEHOLDER_TITLE_GUID); + stakeholder.getRelationshipAttributes().remove(REL_ATTR_STAKEHOLDER_DOMAIN); + stakeholder.getRelationshipAttributes().remove(REL_ATTR_STAKEHOLDER_TITLE); + + + String currentName = vertex.getProperty(NAME, String.class); + String newName = (String) stakeholder.getAttribute(NAME); + + if (!currentName.equals(newName)) { + verifyDuplicateAssetByName(STAKEHOLDER_ENTITY_TYPE, newName, discovery, + String.format("Stakeholder with name %s already exists for current domain", newName)); } String vertexQName = vertex.getProperty(QUALIFIED_NAME, String.class); @@ -228,4 +251,58 @@ private String getQualifiedNameFromRelationAttribute(AtlasEntity entity, String return qualifiedName; } + + protected void verifyDuplicateByDomainAndTitle(String domainQualifiedName, String stakeholderTitleGuid) throws AtlasBaseException { + + List> mustClauseList = new ArrayList(); + mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", STAKEHOLDER_ENTITY_TYPE))); + mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); + mustClauseList.add(mapOf("term", mapOf("domainQualifiedName", domainQualifiedName))); + mustClauseList.add(mapOf("term", mapOf("stakeholderTitleGuid", stakeholderTitleGuid))); + + + Map bool = mapOf("must", mustClauseList); + + Map dsl = mapOf("query", mapOf("bool", bool)); + + List assets = indexSearchPaginated(dsl, null, this.discovery); + + if (CollectionUtils.isNotEmpty(assets)) { + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, + String.format("Stakeholder for provided title & domain combination already exists")); + } + } + + protected void ensureTitleAvailableForDomain(String domainQualifiedName, String stakeholderTitleGuid) throws AtlasBaseException { + + List> mustClauseList = new ArrayList(); + mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", STAKEHOLDER_TITLE_ENTITY_TYPE))); + mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); + mustClauseList.add(mapOf("term", mapOf("__guid", stakeholderTitleGuid))); + + /*List> shouldClauseList = new ArrayList(); + shouldClauseList.add(mapOf("term", mapOf("domainQualifiedNames", domainQualifiedName))); + shouldClauseList.add(mapOf("term", mapOf("domainQualifiedNames", "*"))); + mustClauseList.add(mapOf("bool", mapOf("should", shouldClauseList))); + */ + + Map bool = mapOf("must", mustClauseList); + + Map dsl = mapOf("query", mapOf("bool", bool)); + + List assets = indexSearchPaginated(dsl, null, this.discovery); + + if (CollectionUtils.isNotEmpty(assets)) { + AtlasEntityHeader stakeholderTitleHeader = assets.get(0); + + List domainQualifiedNames = (List) stakeholderTitleHeader.getAttribute(ATTR_DOMAIN_QUALIFIED_NAMES); + + Optional parentDomain = domainQualifiedNames.stream().filter(x -> domainQualifiedName.startsWith(x)).findFirst(); + + if (!parentDomain.isPresent()) { + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, + String.format("Provided StakeholderTitle is not applicable to the domain")); + } + } + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java index 1fe3e35e9ff..ca68510d953 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java @@ -1,14 +1,15 @@ package org.apache.atlas.repository.store.graph.v2.preprocessor.datamesh; import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.AtlasException; import org.apache.atlas.RequestContext; import org.apache.atlas.authorize.AtlasAuthorizationUtils; import org.apache.atlas.authorize.AtlasEntityAccessRequest; import org.apache.atlas.authorize.AtlasPrivilege; +import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntityHeader; -import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.instance.AtlasRelatedObjectId; import org.apache.atlas.model.instance.AtlasRelationship; import org.apache.atlas.model.instance.AtlasStruct; @@ -16,19 +17,19 @@ import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.AtlasEntityStore; -import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; -import org.apache.atlas.repository.store.users.KeycloakStore; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; -import org.apache.commons.lang.StringUtils; +import org.apache.commons.collections.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.ArrayList; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; @@ -36,17 +37,31 @@ import static org.apache.atlas.repository.Constants.DATA_DOMAIN_ENTITY_TYPE; import static org.apache.atlas.repository.Constants.NAME; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; +import static org.apache.atlas.repository.Constants.STAKEHOLDER_ENTITY_TYPE; +import static org.apache.atlas.repository.Constants.STAKEHOLDER_TITLE_ENTITY_TYPE; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.getUUID; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.indexSearchPaginated; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.verifyDuplicateAssetByName; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; public class StakeholderTitlePreProcessor implements PreProcessor { private static final Logger LOG = LoggerFactory.getLogger(StakeholderTitlePreProcessor.class); + public static final String PATTERN_QUALIFIED_NAME_ALL_DOMAINS = "stakeholderTitle/domain/default/%s"; + public static final String PATTERN_QUALIFIED_NAME_DOMAIN = "stakeholderTitle/domain/%s"; + + + public static final String STAR = "*"; + public static final String ATTR_DOMAIN_QUALIFIED_NAMES = "domainQualifiedNames"; + + public static final String REL_ATTR_STAKEHOLDERS = "stakeholders"; + private final AtlasGraph graph; private final AtlasTypeRegistry typeRegistry; private final EntityGraphRetriever entityRetriever; private AtlasEntityStore entityStore; + protected EntityDiscoveryService discovery; public StakeholderTitlePreProcessor(AtlasGraph graph, AtlasTypeRegistry typeRegistry, @@ -56,6 +71,12 @@ public StakeholderTitlePreProcessor(AtlasGraph graph, this.typeRegistry = typeRegistry; this.entityRetriever = entityRetriever; this.entityStore = entityStore; + + try { + this.discovery = new EntityDiscoveryService(typeRegistry, graph, null, null, null, null); + } catch (AtlasException e) { + e.printStackTrace(); + } } @Override @@ -81,7 +102,7 @@ private void processCreateStakeholderTitle(AtlasEntity entity) throws AtlasBaseE AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreateStakeholderTitle"); try { - if (entity.hasRelationshipAttribute("stakeholders")) { + if (entity.hasRelationshipAttribute(REL_ATTR_STAKEHOLDERS)) { throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Managing Stakeholders while creating StakeholderTitle"); } @@ -89,10 +110,14 @@ private void processCreateStakeholderTitle(AtlasEntity entity) throws AtlasBaseE return; } + String name = (String) entity.getAttribute(NAME); + verifyDuplicateAssetByName(STAKEHOLDER_TITLE_ENTITY_TYPE, name, discovery, + String.format("Stakeholder title with name %s already exists", name)); + List domainQualifiedNames = null; - if (entity.hasAttribute("domainQualifiedNames")) { - domainQualifiedNames = (List) entity.getAttribute("domainQualifiedNames"); + if (entity.hasAttribute(ATTR_DOMAIN_QUALIFIED_NAMES)) { + domainQualifiedNames = (List) entity.getAttribute(ATTR_DOMAIN_QUALIFIED_NAMES); if (domainQualifiedNames.size() == 0) { throw new AtlasBaseException(BAD_REQUEST, "Please pass attribute domainQualifiedNames"); } @@ -101,20 +126,19 @@ private void processCreateStakeholderTitle(AtlasEntity entity) throws AtlasBaseE throw new AtlasBaseException(BAD_REQUEST, "Please pass attribute domainQualifiedNames"); } - if ((domainQualifiedNames.size() == 1 && "*".equals(domainQualifiedNames.get(0))) - || domainQualifiedNames.contains("*")) { + if ((domainQualifiedNames.size() == 1 && STAR.equals(domainQualifiedNames.get(0))) + || domainQualifiedNames.contains(STAR)) { AtlasEntityHeader allDomainEntityHeader = new AtlasEntityHeader(DATA_DOMAIN_ENTITY_TYPE, mapOf(QUALIFIED_NAME, "*/super")); AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, new AtlasEntityHeader(allDomainEntityHeader)), "create StakeholderTitle for all domains"); - String qualifiedName = String.format("stakeholderTitle/domain/default/%s", getUUID()); + String qualifiedName = String.format(PATTERN_QUALIFIED_NAME_ALL_DOMAINS, getUUID()); entity.setAttribute(QUALIFIED_NAME, qualifiedName); - entity.setAttribute("domainQualifiedNames", Collections.singletonList("*")); + entity.setAttribute(ATTR_DOMAIN_QUALIFIED_NAMES, Collections.singletonList(STAR)); } else { for (String domainQualifiedName : domainQualifiedNames) { - //AtlasVertex domainVertex = entityRetriever.getEntityVertex(new AtlasObjectId(DATA_DOMAIN_ENTITY_TYPE, mapOf(QUALIFIED_NAME, domainQualifiedName))); - //AtlasEntityHeader domainHeader = entityRetriever.toAtlasEntityHeader(domainVertex); + AtlasEntityHeader domainHeader = new AtlasEntityHeader(DATA_DOMAIN_ENTITY_TYPE, mapOf(QUALIFIED_NAME, domainQualifiedName)); String qualifiedName = String.format("stakeholderTitle/domain/%s/%s", getUUID(), domainQualifiedName); @@ -124,7 +148,7 @@ private void processCreateStakeholderTitle(AtlasEntity entity) throws AtlasBaseE "create StakeholderTitle for domain ", domainQualifiedName); } - entity.setAttribute(QUALIFIED_NAME, String.format("stakeholderTitle/domain/%s", getUUID())); + entity.setAttribute(QUALIFIED_NAME, String.format(PATTERN_QUALIFIED_NAME_DOMAIN, getUUID())); } } finally { @@ -140,11 +164,19 @@ private void processUpdateStakeholderTitle(EntityMutationContext context, AtlasE return; } - if (entity.hasRelationshipAttribute("stakeholders")) { + if (entity.hasRelationshipAttribute(REL_ATTR_STAKEHOLDERS)) { throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Managing Stakeholders while updating StakeholderTitle"); } AtlasVertex vertex = context.getVertex(entity.getGuid()); + + String currentName = vertex.getProperty(NAME, String.class); + String newName = (String) entity.getAttribute(NAME); + if (!currentName.equals(newName)) { + verifyDuplicateAssetByName(STAKEHOLDER_TITLE_ENTITY_TYPE, newName, discovery, + String.format("Stakeholder title with name %s already exists", newName)); + } + String vertexQName = vertex.getProperty(QUALIFIED_NAME, String.class); entity.setAttribute(QUALIFIED_NAME, vertexQName); @@ -162,7 +194,7 @@ public void processDelete(AtlasVertex vertex) throws AtlasBaseException { try { AtlasEntity titleEntity = entityRetriever.toAtlasEntity(vertex); - List stakeholders = (List) titleEntity.getRelationshipAttribute("stakeholders"); + List stakeholders = (List) titleEntity.getRelationshipAttribute(REL_ATTR_STAKEHOLDERS); Optional activeStakeholder = stakeholders.stream().filter(x -> x.getRelationshipStatus() == AtlasRelationship.Status.ACTIVE).findFirst(); if (activeStakeholder.isPresent()) { From 91e68f53f20323774ed5161bce1dbf4e43469d94 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Fri, 10 May 2024 16:43:41 +0530 Subject: [PATCH 158/316] DG-1235 Refactorings --- .../atlas/repository/graph/GraphHelper.java | 1 - .../v2/DataDomainQNMigrationService.java | 161 ++++++++++-------- .../v2/preprocessor/PreProcessorUtils.java | 1 - .../apache/atlas/web/rest/MigrationREST.java | 29 ++-- 4 files changed, 107 insertions(+), 85 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index 0d403c981f2..7907be09d40 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -1946,7 +1946,6 @@ public static Iterator getAllVertices(AtlasVertex vertex, String ch } } - private static Set parseLabelsString(String labels) { Set ret = new HashSet<>(); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataDomainQNMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataDomainQNMigrationService.java index 7e6a204874a..50b134eb159 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataDomainQNMigrationService.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataDomainQNMigrationService.java @@ -23,7 +23,6 @@ import org.springframework.stereotype.Component; import javax.inject.Inject; -import javax.inject.Named; import java.util.*; import static org.apache.atlas.repository.Constants.*; @@ -47,11 +46,11 @@ public class DataDomainQNMigrationService implements MigrationService{ private final int BATCH_SIZE = 20; - boolean errorOccur = false; + boolean errorOccured = false; boolean skipSuperDomain = false; - private int Counter; + private int counter; private final TransactionInterceptHelper transactionInterceptHelper; private final RedisService redisService; @@ -63,71 +62,78 @@ public DataDomainQNMigrationService(AtlasEntityStore entityStore, RedisService r this.discovery = discovery; this.typeRegistry = typeRegistry; this.updatedPolicyResources = new HashMap<>(); - this.Counter = 0; + this.counter = 0; this.transactionInterceptHelper = transactionInterceptHelper; this.redisService = redisService; } @Async - public Boolean startMigration () throws Exception{ - redisService.putValue(DATA_MESH_QN,IN_PROGRESS); - Set attributes = new HashSet<>(Arrays.asList(QUALIFIED_NAME, SUPER_DOMAIN_QN_ATTR, PARENT_DOMAIN_QN_ATTR, "__customAttributes")); - List entities = null; - entities = getEntity(DATA_DOMAIN_ENTITY_TYPE, attributes, null); + public Boolean startMigration() throws Exception{ + redisService.putValue(DATA_MESH_QN, IN_PROGRESS); + Set attributes = new HashSet<>(Arrays.asList(SUPER_DOMAIN_QN_ATTR, PARENT_DOMAIN_QN_ATTR, "__customAttributes")); - for (int i = 0; i < entities.size(); i++) { + List entities = getEntity(DATA_DOMAIN_ENTITY_TYPE, attributes, null); + + for (AtlasEntityHeader superDomain: entities) { skipSuperDomain = false; - updateChunk(entities.get(i)); + updateChunk(superDomain); + } + + if(errorOccured) { + redisService.putValue(DATA_MESH_QN, FAILED); + } else { + redisService.putValue(DATA_MESH_QN, SUCCESSFUL); } - if(errorOccur) - redisService.putValue(DATA_MESH_QN,FAILED); - else - redisService.putValue(DATA_MESH_QN,SUCCESSFUL); - LOG.info("MIGRATION_RESULT",redisService.getValue("MIGRATION_DATA_DOMAIN_QN")); + + LOG.info("MIGRATION_RESULT {} ",redisService.getValue(DATA_MESH_QN)); return Boolean.TRUE; } - private void updateChunk (AtlasEntityHeader atlasEntity) throws AtlasBaseException { + private void updateChunk(AtlasEntityHeader atlasEntity) throws AtlasBaseException { AtlasVertex vertex = entityRetriever.getEntityVertex(atlasEntity.getGuid()); String qualifiedName = (String) atlasEntity.getAttribute(QUALIFIED_NAME); try{ migrateDomainAttributes(vertex, "", ""); - } - catch (AtlasBaseException e){ - this.errorOccur = true; - LOG.error("Error while migrating qualified name for entity: {}", qualifiedName, e); - } - finally { + commitChanges(); LOG.info("Migrated qualified name for entity: {}", qualifiedName); + + } catch (AtlasBaseException e){ + this.errorOccured = true; + LOG.error("Error while migrating qualified name for entity: {}", qualifiedName, e); } } + private void migrateDomainAttributes(AtlasVertex vertex, String parentDomainQualifiedName, String superDomainQualifiedName) throws AtlasBaseException { + if(skipSuperDomain) { + return; + } + counter++; - private void migrateDomainAttributes(AtlasVertex vertex, String parentDomainQualifiedName, String rootDomainQualifiedName) throws AtlasBaseException { - if(skipSuperDomain) - return; String currentQualifiedName = vertex.getProperty(QUALIFIED_NAME,String.class); + String updatedQualifiedName = createDomainQualifiedName(parentDomainQualifiedName); + LOG.info("Migrating qualified name for Domain: {}", currentQualifiedName); - Counter++; + Map updatedAttributes = new HashMap<>(); - String updatedQualifiedName = createDomainQualifiedName(parentDomainQualifiedName); + Map customAttributes = GraphHelper.getCustomAttributes(vertex); if(customAttributes != null && customAttributes.get(MIGRATION_CUSTOM_ATTRIBUTE) != null && customAttributes.get(MIGRATION_CUSTOM_ATTRIBUTE).equals("true")){ LOG.info("Entity already migrated for entity: {}", currentQualifiedName); + + } else { + superDomainQualifiedName = commitChangesInMemory(currentQualifiedName, updatedQualifiedName, parentDomainQualifiedName, superDomainQualifiedName, vertex, updatedAttributes); } - else{ - rootDomainQualifiedName = commitChangesInMemory(currentQualifiedName,updatedQualifiedName,parentDomainQualifiedName,rootDomainQualifiedName,vertex,updatedAttributes); - } - if(!skipSuperDomain) { + + if (!skipSuperDomain) { Iterator products = getAllChildrenVertices(vertex, DATA_PRODUCT_EDGE_LABEL); while (products.hasNext()) { AtlasVertex productVertex = products.next(); - migrateDataProductAttributes(productVertex, updatedQualifiedName, rootDomainQualifiedName); + migrateDataProductAttributes(productVertex, updatedQualifiedName, superDomainQualifiedName); if(skipSuperDomain) break; } @@ -137,45 +143,52 @@ private void migrateDomainAttributes(AtlasVertex vertex, String parentDomainQual while (childDomains.hasNext()) { AtlasVertex childVertex = childDomains.next(); - migrateDomainAttributes(childVertex, updatedQualifiedName, rootDomainQualifiedName); + migrateDomainAttributes(childVertex, updatedQualifiedName, superDomainQualifiedName); if(skipSuperDomain) break; } recordUpdatedChildEntities(vertex, updatedAttributes); - if (Counter >= BATCH_SIZE) { + if (counter >= BATCH_SIZE) { commitChanges(); } } - } - public void commitChanges(){ + public void commitChanges() { try { updatePolicy(this.updatedPolicyResources); + this.updatedPolicyResources.clear(); } catch (AtlasBaseException e) { - this.errorOccur = true; + this.errorOccured = true; + this.skipSuperDomain = true; throw new RuntimeException(e); } + try { - transactionInterceptHelper.intercept(); - }catch (Exception e){ + if(!skipSuperDomain) { + transactionInterceptHelper.intercept(); + this.counter = 0; + } + } catch (Exception e){ this.skipSuperDomain = true; - this.errorOccur = true; + this.errorOccured = true; } - this.updatedPolicyResources.clear(); - this.Counter = 0; } - public String commitChangesInMemory(String currentQualifiedName, String updatedQualifiedName, String parentDomainQualifiedName, String rootDomainQualifiedName, AtlasVertex vertex, Map updatedAttributes){ - if(skipSuperDomain) + + public String commitChangesInMemory(String currentQualifiedName, String updatedQualifiedName, String parentDomainQualifiedName, String superDomainQualifiedName, AtlasVertex vertex, Map updatedAttributes) { + + if(skipSuperDomain) { return ""; - vertex.setProperty(QUALIFIED_NAME, updatedQualifiedName); - if (StringUtils.isEmpty(parentDomainQualifiedName) && StringUtils.isEmpty(rootDomainQualifiedName)){ - rootDomainQualifiedName = updatedQualifiedName; } - else{ + + vertex.setProperty(QUALIFIED_NAME, updatedQualifiedName); + + if (StringUtils.isEmpty(parentDomainQualifiedName) && StringUtils.isEmpty(superDomainQualifiedName)){ + superDomainQualifiedName = updatedQualifiedName; + } else{ vertex.setProperty(PARENT_DOMAIN_QN_ATTR, parentDomainQualifiedName); - vertex.setProperty(SUPER_DOMAIN_QN_ATTR, rootDomainQualifiedName); + vertex.setProperty(SUPER_DOMAIN_QN_ATTR, superDomainQualifiedName); } updatedAttributes.put(QUALIFIED_NAME, updatedQualifiedName); @@ -184,28 +197,35 @@ public String commitChangesInMemory(String currentQualifiedName, String updatedQ String currentResource = "entity:"+ currentQualifiedName; String updatedResource = "entity:"+ updatedQualifiedName; this.updatedPolicyResources.put(currentResource, updatedResource); + Map customAttributes = GraphHelper.getCustomAttributes(vertex); - if(Objects.isNull(customAttributes) || MapUtils.isEmpty(customAttributes)) + if(Objects.isNull(customAttributes) || MapUtils.isEmpty(customAttributes)) { customAttributes = new HashMap<>(); + } customAttributes.put(MIGRATION_CUSTOM_ATTRIBUTE, "true"); vertex.setProperty(CUSTOM_ATTRIBUTES_PROPERTY_KEY, AtlasEntityType.toJson(customAttributes)); - return rootDomainQualifiedName; + return superDomainQualifiedName; } - private void migrateDataProductAttributes(AtlasVertex vertex, String parentDomainQualifiedName, String rootDomainQualifiedName) throws AtlasBaseException { - if(skipSuperDomain) + private void migrateDataProductAttributes(AtlasVertex vertex, String parentDomainQualifiedName, String superDomainQualifiedName) throws AtlasBaseException { + if(skipSuperDomain) { return; - Counter++; - String currentQualifiedName = (String) vertex.getProperty(QUALIFIED_NAME,String.class); - LOG.info("Migrating qualified name for Product: {}", currentQualifiedName); - Map customAttributes_ = GraphHelper.getCustomAttributes(vertex); - String updatedQualifiedName = createProductQualifiedName(parentDomainQualifiedName); - if(customAttributes_ != null && customAttributes_.get(MIGRATION_CUSTOM_ATTRIBUTE) != null && customAttributes_.get(MIGRATION_CUSTOM_ATTRIBUTE).equals("true")){ - LOG.info("Entity already migrated for entity: {}", currentQualifiedName); } - else { + + counter++; + + String currentQualifiedName = vertex.getProperty(QUALIFIED_NAME,String.class); + String updatedQualifiedName = createProductQualifiedName(parentDomainQualifiedName); + + LOG.info("Migrating qualified name for Product: {}", currentQualifiedName); + Map customAttributes = GraphHelper.getCustomAttributes(vertex); + + if(customAttributes != null && customAttributes.get(MIGRATION_CUSTOM_ATTRIBUTE) != null && customAttributes.get(MIGRATION_CUSTOM_ATTRIBUTE).equals("true")) { + LOG.info("Product already migrated: {}", currentQualifiedName); + + } else { vertex.setProperty(QUALIFIED_NAME, updatedQualifiedName); //Store domainPolicies and resources to be updated @@ -214,22 +234,25 @@ private void migrateDataProductAttributes(AtlasVertex vertex, String parentDomai this.updatedPolicyResources.put(currentResource, updatedResource); vertex.setProperty(PARENT_DOMAIN_QN_ATTR, parentDomainQualifiedName); - vertex.setProperty(SUPER_DOMAIN_QN_ATTR, rootDomainQualifiedName); + vertex.setProperty(SUPER_DOMAIN_QN_ATTR, superDomainQualifiedName); - Map customAttributes = GraphHelper.getCustomAttributes(vertex); - if(Objects.isNull(customAttributes) || MapUtils.isEmpty(customAttributes)) + if(Objects.isNull(customAttributes) || MapUtils.isEmpty(customAttributes)) { customAttributes = new HashMap<>(); + } customAttributes.put(MIGRATION_CUSTOM_ATTRIBUTE, "true"); vertex.setProperty(CUSTOM_ATTRIBUTES_PROPERTY_KEY, AtlasEntityType.toJson(customAttributes)); } - if(Counter >= BATCH_SIZE){ + + if(counter >= BATCH_SIZE){ commitChanges(); } } protected void updatePolicy(Map updatedPolicyResources) throws AtlasBaseException { - if(skipSuperDomain) + if(skipSuperDomain) { return; + } + List currentResources = new ArrayList<>(updatedPolicyResources.keySet()); LOG.info("Updating policies for entities {}", currentResources); Map updatedAttributes = new HashMap<>(); @@ -267,9 +290,7 @@ protected void updatePolicy(Map updatedPolicyResources) throws A EntityStream entityStream = new AtlasEntityStream(entityList); entityStore.createOrUpdate(entityStream, false); } - } - } private static String createDomainQualifiedName(String parentDomainQualifiedName) { @@ -392,7 +413,7 @@ protected void recordUpdatedChildEntities(AtlasVertex entityVertex, Map Date: Fri, 10 May 2024 17:35:09 +0530 Subject: [PATCH 159/316] DG-1235 Fix title update 403 --- .../StakeholderTitlePreProcessor.java | 25 +++++++++++++++---- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java index ca68510d953..57e195c9b46 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java @@ -130,7 +130,7 @@ private void processCreateStakeholderTitle(AtlasEntity entity) throws AtlasBaseE || domainQualifiedNames.contains(STAR)) { AtlasEntityHeader allDomainEntityHeader = new AtlasEntityHeader(DATA_DOMAIN_ENTITY_TYPE, mapOf(QUALIFIED_NAME, "*/super")); - AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, new AtlasEntityHeader(allDomainEntityHeader)), + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, new AtlasEntityHeader(allDomainEntityHeader)), "create StakeholderTitle for all domains"); String qualifiedName = String.format(PATTERN_QUALIFIED_NAME_ALL_DOMAINS, getUUID()); @@ -140,12 +140,11 @@ private void processCreateStakeholderTitle(AtlasEntity entity) throws AtlasBaseE for (String domainQualifiedName : domainQualifiedNames) { AtlasEntityHeader domainHeader = new AtlasEntityHeader(DATA_DOMAIN_ENTITY_TYPE, mapOf(QUALIFIED_NAME, domainQualifiedName)); - String qualifiedName = String.format("stakeholderTitle/domain/%s/%s", getUUID(), domainQualifiedName); + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, new AtlasEntityHeader(domainHeader)), + "create StakeholderTitle for domain ", domainQualifiedName); + String qualifiedName = String.format("stakeholderTitle/domain/%s/%s", getUUID(), domainQualifiedName); entity.setAttribute(QUALIFIED_NAME, qualifiedName); - - AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, new AtlasEntityHeader(domainHeader)), - "create StakeholderTitle for domain ", domainQualifiedName); } entity.setAttribute(QUALIFIED_NAME, String.format(PATTERN_QUALIFIED_NAME_DOMAIN, getUUID())); @@ -177,6 +176,22 @@ private void processUpdateStakeholderTitle(EntityMutationContext context, AtlasE String.format("Stakeholder title with name %s already exists", newName)); } + List currentDomainQualifiedNames; + + if (entity.hasAttribute(ATTR_DOMAIN_QUALIFIED_NAMES)) { + // updating domains list authorize only on all domains in the list + currentDomainQualifiedNames = (List) entity.getAttribute(ATTR_DOMAIN_QUALIFIED_NAMES); + } else { + // updating other metadata, authorize update on all existing domains + currentDomainQualifiedNames = vertex.getListProperty(ATTR_DOMAIN_QUALIFIED_NAMES, String.class); + } + + for (String domainQualifiedName: currentDomainQualifiedNames) { + AtlasEntityHeader domainHeader = new AtlasEntityHeader(DATA_DOMAIN_ENTITY_TYPE, mapOf(QUALIFIED_NAME, domainQualifiedName)); + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, new AtlasEntityHeader(domainHeader)), + "create StakeholderTitle for domain ", domainQualifiedName); + } + String vertexQName = vertex.getProperty(QUALIFIED_NAME, String.class); entity.setAttribute(QUALIFIED_NAME, vertexQName); From bd6673881212abf6f25db75a63b6b71ea9ef50ba Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Fri, 10 May 2024 18:13:49 +0530 Subject: [PATCH 160/316] DG-1235 Refactorings --- .../apache/atlas/repository/Constants.java | 1 + .../StakeholderTitlePreProcessor.java | 61 ++++++++++--------- 2 files changed, 34 insertions(+), 28 deletions(-) diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index b5fd8a85194..8fb5327a7d0 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -448,6 +448,7 @@ public enum SupportedFileExtensions { XLSX, XLS, CSV } add(README_ENTITY_TYPE); add(LINK_ENTITY_TYPE); add(POLICY_ENTITY_TYPE); + add(STAKEHOLDER_TITLE_ENTITY_TYPE); }}; private Constants() { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java index 57e195c9b46..ba929b5cd7f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java @@ -126,30 +126,22 @@ private void processCreateStakeholderTitle(AtlasEntity entity) throws AtlasBaseE throw new AtlasBaseException(BAD_REQUEST, "Please pass attribute domainQualifiedNames"); } - if ((domainQualifiedNames.size() == 1 && STAR.equals(domainQualifiedNames.get(0))) - || domainQualifiedNames.contains(STAR)) { - - AtlasEntityHeader allDomainEntityHeader = new AtlasEntityHeader(DATA_DOMAIN_ENTITY_TYPE, mapOf(QUALIFIED_NAME, "*/super")); - AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, new AtlasEntityHeader(allDomainEntityHeader)), - "create StakeholderTitle for all domains"); + if (domainQualifiedNames.contains(STAR)) { + if (domainQualifiedNames.size() > 1) { + domainQualifiedNames.clear(); + domainQualifiedNames.add(STAR); + entity.setAttribute(ATTR_DOMAIN_QUALIFIED_NAMES, domainQualifiedNames); + } String qualifiedName = String.format(PATTERN_QUALIFIED_NAME_ALL_DOMAINS, getUUID()); entity.setAttribute(QUALIFIED_NAME, qualifiedName); - entity.setAttribute(ATTR_DOMAIN_QUALIFIED_NAMES, Collections.singletonList(STAR)); - } else { - for (String domainQualifiedName : domainQualifiedNames) { - - AtlasEntityHeader domainHeader = new AtlasEntityHeader(DATA_DOMAIN_ENTITY_TYPE, mapOf(QUALIFIED_NAME, domainQualifiedName)); - AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, new AtlasEntityHeader(domainHeader)), - "create StakeholderTitle for domain ", domainQualifiedName); - - String qualifiedName = String.format("stakeholderTitle/domain/%s/%s", getUUID(), domainQualifiedName); - entity.setAttribute(QUALIFIED_NAME, qualifiedName); - } + } else { entity.setAttribute(QUALIFIED_NAME, String.format(PATTERN_QUALIFIED_NAME_DOMAIN, getUUID())); } + authorizeDomainAccess(domainQualifiedNames, AtlasPrivilege.ENTITY_UPDATE); + } finally { RequestContext.get().endMetricRecord(metricRecorder); } @@ -176,21 +168,14 @@ private void processUpdateStakeholderTitle(EntityMutationContext context, AtlasE String.format("Stakeholder title with name %s already exists", newName)); } - List currentDomainQualifiedNames; - + List domainQualifiedNames; if (entity.hasAttribute(ATTR_DOMAIN_QUALIFIED_NAMES)) { - // updating domains list authorize only on all domains in the list - currentDomainQualifiedNames = (List) entity.getAttribute(ATTR_DOMAIN_QUALIFIED_NAMES); + domainQualifiedNames = (List) entity.getAttribute(ATTR_DOMAIN_QUALIFIED_NAMES); } else { - // updating other metadata, authorize update on all existing domains - currentDomainQualifiedNames = vertex.getListProperty(ATTR_DOMAIN_QUALIFIED_NAMES, String.class); + domainQualifiedNames = vertex.getListProperty(ATTR_DOMAIN_QUALIFIED_NAMES, String.class); } - for (String domainQualifiedName: currentDomainQualifiedNames) { - AtlasEntityHeader domainHeader = new AtlasEntityHeader(DATA_DOMAIN_ENTITY_TYPE, mapOf(QUALIFIED_NAME, domainQualifiedName)); - AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, new AtlasEntityHeader(domainHeader)), - "create StakeholderTitle for domain ", domainQualifiedName); - } + authorizeDomainAccess(domainQualifiedNames, AtlasPrivilege.ENTITY_UPDATE); String vertexQName = vertex.getProperty(QUALIFIED_NAME, String.class); entity.setAttribute(QUALIFIED_NAME, vertexQName); @@ -216,9 +201,29 @@ public void processDelete(AtlasVertex vertex) throws AtlasBaseException { throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Can not delete StakeholderTitle as it has reference to Active Stakeholder"); } + List domainQualifiedNames = vertex.getListProperty(ATTR_DOMAIN_QUALIFIED_NAMES, String.class); + + authorizeDomainAccess(domainQualifiedNames, AtlasPrivilege.ENTITY_UPDATE); + } finally { RequestContext.get().endMetricRecord(metricRecorder); } } + + private void authorizeDomainAccess(List domainQualifiedNames, AtlasPrivilege permission) throws AtlasBaseException { + for (String domainQualifiedName: domainQualifiedNames) { + String domainQualifiedNameToAuth; + if (domainQualifiedNames.contains(STAR)) { + domainQualifiedNameToAuth = "*/super"; + } else { + domainQualifiedNameToAuth = domainQualifiedName; + } + + AtlasEntityHeader domainHeaderToAuth = new AtlasEntityHeader(DATA_DOMAIN_ENTITY_TYPE, mapOf(QUALIFIED_NAME, domainQualifiedNameToAuth)); + + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, new AtlasEntityHeader(domainHeaderToAuth)), + "create StakeholderTitle for domain ", domainQualifiedName); + } + } } From 202c236712ac0eac6f8bfea79b24f9cebd76ec31 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Fri, 10 May 2024 18:20:42 +0530 Subject: [PATCH 161/316] DG-1235 Fix Stakeholder update issue --- .../preprocessor/accesscontrol/StakeholderPreProcessor.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java index 949e5aa7b8e..784f86d2f92 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java @@ -45,6 +45,8 @@ import org.slf4j.LoggerFactory; import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; @@ -290,7 +292,7 @@ protected void ensureTitleAvailableForDomain(String domainQualifiedName, String Map dsl = mapOf("query", mapOf("bool", bool)); - List assets = indexSearchPaginated(dsl, null, this.discovery); + List assets = indexSearchPaginated(dsl, Collections.singleton(ATTR_DOMAIN_QUALIFIED_NAMES), this.discovery); if (CollectionUtils.isNotEmpty(assets)) { AtlasEntityHeader stakeholderTitleHeader = assets.get(0); From 3549cbcc30b87fcf0554fe1c8f2fde8f9455ab27 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Fri, 10 May 2024 18:59:56 +0530 Subject: [PATCH 162/316] Added NPE catches and changed from @async to Runnable logic --- ...e.java => DataMeshQNMigrationService.java} | 49 ++++++++++++++----- .../apache/atlas/web/rest/MigrationREST.java | 32 ++++-------- 2 files changed, 46 insertions(+), 35 deletions(-) rename repository/src/main/java/org/apache/atlas/repository/store/graph/v2/{DataDomainQNMigrationService.java => DataMeshQNMigrationService.java} (90%) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataDomainQNMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java similarity index 90% rename from repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataDomainQNMigrationService.java rename to repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java index 50b134eb159..862c316a831 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataDomainQNMigrationService.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java @@ -1,5 +1,6 @@ package org.apache.atlas.repository.store.graph.v2; +import jline.internal.Log; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.RequestContext; import org.apache.atlas.discovery.EntityDiscoveryService; @@ -19,7 +20,6 @@ import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Component; import javax.inject.Inject; @@ -33,9 +33,9 @@ import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_RESOURCES; @Component -public class DataDomainQNMigrationService implements MigrationService{ +public class DataMeshQNMigrationService implements MigrationService, Runnable { - private static final Logger LOG = LoggerFactory.getLogger(DataDomainQNMigrationService.class); + private static final Logger LOG = LoggerFactory.getLogger(DataMeshQNMigrationService.class); private final AtlasEntityStore entityStore; private final EntityDiscoveryService discovery; @@ -56,7 +56,7 @@ public class DataDomainQNMigrationService implements MigrationService{ private final RedisService redisService; @Inject - public DataDomainQNMigrationService(AtlasEntityStore entityStore, RedisService redisService, EntityDiscoveryService discovery, EntityGraphRetriever entityRetriever, AtlasTypeRegistry typeRegistry, TransactionInterceptHelper transactionInterceptHelper) { + public DataMeshQNMigrationService(AtlasEntityStore entityStore, RedisService redisService, EntityDiscoveryService discovery, EntityGraphRetriever entityRetriever, AtlasTypeRegistry typeRegistry, TransactionInterceptHelper transactionInterceptHelper) { this.entityRetriever = entityRetriever; this.entityStore = entityStore; this.discovery = discovery; @@ -67,7 +67,6 @@ public DataDomainQNMigrationService(AtlasEntityStore entityStore, RedisService r this.redisService = redisService; } - @Async public Boolean startMigration() throws Exception{ redisService.putValue(DATA_MESH_QN, IN_PROGRESS); @@ -132,20 +131,34 @@ private void migrateDomainAttributes(AtlasVertex vertex, String parentDomainQual Iterator products = getAllChildrenVertices(vertex, DATA_PRODUCT_EDGE_LABEL); while (products.hasNext()) { - AtlasVertex productVertex = products.next(); - migrateDataProductAttributes(productVertex, updatedQualifiedName, superDomainQualifiedName); - if(skipSuperDomain) - break; + try { + AtlasVertex productVertex = products.next(); + if (Objects.nonNull(productVertex)) { + migrateDataProductAttributes(productVertex, updatedQualifiedName, superDomainQualifiedName); + } + if (skipSuperDomain) + break; + } + catch (NullPointerException e){ + LOG.error("Null Pointer Exception occured for products with parent : {}",currentQualifiedName); + } } // Get all children domains of current domain Iterator childDomains = getAllChildrenVertices(vertex, DOMAIN_PARENT_EDGE_LABEL); while (childDomains.hasNext()) { - AtlasVertex childVertex = childDomains.next(); - migrateDomainAttributes(childVertex, updatedQualifiedName, superDomainQualifiedName); - if(skipSuperDomain) - break; + try { + AtlasVertex childVertex = childDomains.next(); + if (Objects.nonNull(childVertex)) { + migrateDomainAttributes(childVertex, updatedQualifiedName, superDomainQualifiedName); + } + if (skipSuperDomain) + break; + } + catch (NullPointerException e){ + LOG.error("Null Pointer Exception occured for subdomains with parent : {}",currentQualifiedName); + } } recordUpdatedChildEntities(vertex, updatedAttributes); @@ -430,4 +443,14 @@ public static Map mapOf(String key, Object value) { map.put(key, value); return map; } + + @Override + public void run() { + try { + startMigration(); + } catch (Exception e) { + LOG.error("Error running migration : {}",e.toString()); + throw new RuntimeException(e); + } + } } \ No newline at end of file diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java index 4b6bd8e2d70..d2281add5bc 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java @@ -11,6 +11,7 @@ import org.apache.atlas.repository.graphdb.*; import org.apache.atlas.repository.store.graph.AtlasEntityStore; import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream; +import org.apache.atlas.repository.store.graph.v2.DataMeshQNMigrationService; import org.apache.atlas.repository.store.graph.v2.EntityStream; import org.apache.atlas.repository.store.graph.v2.MigrationService; import org.apache.atlas.repository.store.users.KeycloakStore; @@ -57,22 +58,18 @@ public class MigrationREST { private final PreProcessorPoliciesTransformer transformer; private KeycloakStore keycloakStore; private AtlasGraph graph; - private final Map migrationServicesMap = new HashMap<>(); - List migrationServices; + DataMeshQNMigrationService dataMeshQNMigrationService; private final RedisService redisService; @Inject - public MigrationREST(AtlasEntityStore entityStore, AtlasGraph graph, List migrationServices,RedisService redisService) { + public MigrationREST(AtlasEntityStore entityStore, AtlasGraph graph, DataMeshQNMigrationService dataMeshQNMigrationService,RedisService redisService) { this.entityStore = entityStore; this.graph = graph; this.transformer = new PreProcessorPoliciesTransformer(); - this.migrationServices = migrationServices; keycloakStore = new KeycloakStore(); this.redisService = redisService; - for (MigrationService service : migrationServices) { - String[] path = service.getClass().getName().split("\\."); - migrationServicesMap.put(path[path.length - 1], service); - } + this.dataMeshQNMigrationService = dataMeshQNMigrationService; + } @POST @@ -85,28 +82,19 @@ public Boolean submit (@QueryParam("migrationType") String migrationType) throws if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "MigrationREST.submit()"); } - - migrationService = getMigrationService(MIGRATION + migrationType); - - return Objects.nonNull(migrationService) ? migrationService.startMigration() : Boolean.FALSE; + if( (MIGRATION + migrationType).equals(DATA_MESH_QN) ){ + dataMeshQNMigrationService.run(); + } } catch (Exception e) { LOG.error("Error while submitting migration", e); - throw e; + return Boolean.FALSE; } finally { AtlasPerfTracer.log(perf); } + return Boolean.TRUE; } - private MigrationService getMigrationService(String migrationType){ - switch (migrationType){ - case DATA_MESH_QN: - return migrationServicesMap.get("DataDomainQNMigrationService"); - default: - LOG.warn("No service type found"); - return null; - } - } @GET @Path("status") From 4fdef9f567b5cf785748a84da8df17904ec8b824 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Fri, 10 May 2024 18:59:56 +0530 Subject: [PATCH 163/316] Added NPE catches and changed from @async to Runnable logic --- ...e.java => DataMeshQNMigrationService.java} | 49 ++++++++++++++----- .../apache/atlas/web/rest/MigrationREST.java | 32 ++++-------- 2 files changed, 46 insertions(+), 35 deletions(-) rename repository/src/main/java/org/apache/atlas/repository/store/graph/v2/{DataDomainQNMigrationService.java => DataMeshQNMigrationService.java} (90%) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataDomainQNMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java similarity index 90% rename from repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataDomainQNMigrationService.java rename to repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java index 50b134eb159..862c316a831 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataDomainQNMigrationService.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java @@ -1,5 +1,6 @@ package org.apache.atlas.repository.store.graph.v2; +import jline.internal.Log; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.RequestContext; import org.apache.atlas.discovery.EntityDiscoveryService; @@ -19,7 +20,6 @@ import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.scheduling.annotation.Async; import org.springframework.stereotype.Component; import javax.inject.Inject; @@ -33,9 +33,9 @@ import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_RESOURCES; @Component -public class DataDomainQNMigrationService implements MigrationService{ +public class DataMeshQNMigrationService implements MigrationService, Runnable { - private static final Logger LOG = LoggerFactory.getLogger(DataDomainQNMigrationService.class); + private static final Logger LOG = LoggerFactory.getLogger(DataMeshQNMigrationService.class); private final AtlasEntityStore entityStore; private final EntityDiscoveryService discovery; @@ -56,7 +56,7 @@ public class DataDomainQNMigrationService implements MigrationService{ private final RedisService redisService; @Inject - public DataDomainQNMigrationService(AtlasEntityStore entityStore, RedisService redisService, EntityDiscoveryService discovery, EntityGraphRetriever entityRetriever, AtlasTypeRegistry typeRegistry, TransactionInterceptHelper transactionInterceptHelper) { + public DataMeshQNMigrationService(AtlasEntityStore entityStore, RedisService redisService, EntityDiscoveryService discovery, EntityGraphRetriever entityRetriever, AtlasTypeRegistry typeRegistry, TransactionInterceptHelper transactionInterceptHelper) { this.entityRetriever = entityRetriever; this.entityStore = entityStore; this.discovery = discovery; @@ -67,7 +67,6 @@ public DataDomainQNMigrationService(AtlasEntityStore entityStore, RedisService r this.redisService = redisService; } - @Async public Boolean startMigration() throws Exception{ redisService.putValue(DATA_MESH_QN, IN_PROGRESS); @@ -132,20 +131,34 @@ private void migrateDomainAttributes(AtlasVertex vertex, String parentDomainQual Iterator products = getAllChildrenVertices(vertex, DATA_PRODUCT_EDGE_LABEL); while (products.hasNext()) { - AtlasVertex productVertex = products.next(); - migrateDataProductAttributes(productVertex, updatedQualifiedName, superDomainQualifiedName); - if(skipSuperDomain) - break; + try { + AtlasVertex productVertex = products.next(); + if (Objects.nonNull(productVertex)) { + migrateDataProductAttributes(productVertex, updatedQualifiedName, superDomainQualifiedName); + } + if (skipSuperDomain) + break; + } + catch (NullPointerException e){ + LOG.error("Null Pointer Exception occured for products with parent : {}",currentQualifiedName); + } } // Get all children domains of current domain Iterator childDomains = getAllChildrenVertices(vertex, DOMAIN_PARENT_EDGE_LABEL); while (childDomains.hasNext()) { - AtlasVertex childVertex = childDomains.next(); - migrateDomainAttributes(childVertex, updatedQualifiedName, superDomainQualifiedName); - if(skipSuperDomain) - break; + try { + AtlasVertex childVertex = childDomains.next(); + if (Objects.nonNull(childVertex)) { + migrateDomainAttributes(childVertex, updatedQualifiedName, superDomainQualifiedName); + } + if (skipSuperDomain) + break; + } + catch (NullPointerException e){ + LOG.error("Null Pointer Exception occured for subdomains with parent : {}",currentQualifiedName); + } } recordUpdatedChildEntities(vertex, updatedAttributes); @@ -430,4 +443,14 @@ public static Map mapOf(String key, Object value) { map.put(key, value); return map; } + + @Override + public void run() { + try { + startMigration(); + } catch (Exception e) { + LOG.error("Error running migration : {}",e.toString()); + throw new RuntimeException(e); + } + } } \ No newline at end of file diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java index 4b6bd8e2d70..d2281add5bc 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java @@ -11,6 +11,7 @@ import org.apache.atlas.repository.graphdb.*; import org.apache.atlas.repository.store.graph.AtlasEntityStore; import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream; +import org.apache.atlas.repository.store.graph.v2.DataMeshQNMigrationService; import org.apache.atlas.repository.store.graph.v2.EntityStream; import org.apache.atlas.repository.store.graph.v2.MigrationService; import org.apache.atlas.repository.store.users.KeycloakStore; @@ -57,22 +58,18 @@ public class MigrationREST { private final PreProcessorPoliciesTransformer transformer; private KeycloakStore keycloakStore; private AtlasGraph graph; - private final Map migrationServicesMap = new HashMap<>(); - List migrationServices; + DataMeshQNMigrationService dataMeshQNMigrationService; private final RedisService redisService; @Inject - public MigrationREST(AtlasEntityStore entityStore, AtlasGraph graph, List migrationServices,RedisService redisService) { + public MigrationREST(AtlasEntityStore entityStore, AtlasGraph graph, DataMeshQNMigrationService dataMeshQNMigrationService,RedisService redisService) { this.entityStore = entityStore; this.graph = graph; this.transformer = new PreProcessorPoliciesTransformer(); - this.migrationServices = migrationServices; keycloakStore = new KeycloakStore(); this.redisService = redisService; - for (MigrationService service : migrationServices) { - String[] path = service.getClass().getName().split("\\."); - migrationServicesMap.put(path[path.length - 1], service); - } + this.dataMeshQNMigrationService = dataMeshQNMigrationService; + } @POST @@ -85,28 +82,19 @@ public Boolean submit (@QueryParam("migrationType") String migrationType) throws if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "MigrationREST.submit()"); } - - migrationService = getMigrationService(MIGRATION + migrationType); - - return Objects.nonNull(migrationService) ? migrationService.startMigration() : Boolean.FALSE; + if( (MIGRATION + migrationType).equals(DATA_MESH_QN) ){ + dataMeshQNMigrationService.run(); + } } catch (Exception e) { LOG.error("Error while submitting migration", e); - throw e; + return Boolean.FALSE; } finally { AtlasPerfTracer.log(perf); } + return Boolean.TRUE; } - private MigrationService getMigrationService(String migrationType){ - switch (migrationType){ - case DATA_MESH_QN: - return migrationServicesMap.get("DataDomainQNMigrationService"); - default: - LOG.warn("No service type found"); - return null; - } - } @GET @Path("status") From 85e4c75f513c5eac0ad686a19799f297565b4d9b Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Fri, 10 May 2024 19:15:00 +0530 Subject: [PATCH 164/316] DG-1235 Title update 403 --- .../v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java index ba929b5cd7f..b0d4d8aa06e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java @@ -180,9 +180,6 @@ private void processUpdateStakeholderTitle(EntityMutationContext context, AtlasE String vertexQName = vertex.getProperty(QUALIFIED_NAME, String.class); entity.setAttribute(QUALIFIED_NAME, vertexQName); - AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, new AtlasEntityHeader(entity)), - "update StakeholderTitle: ", entity.getAttribute(NAME)); - } finally { RequestContext.get().endMetricRecord(metricRecorder); } From 4a9651f0654fe242b0baefba19964faeedd7a4a3 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Fri, 10 May 2024 19:24:06 +0530 Subject: [PATCH 165/316] DG-1235 Title delete error --- .../v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java index b0d4d8aa06e..185f6110499 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java @@ -198,7 +198,7 @@ public void processDelete(AtlasVertex vertex) throws AtlasBaseException { throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Can not delete StakeholderTitle as it has reference to Active Stakeholder"); } - List domainQualifiedNames = vertex.getListProperty(ATTR_DOMAIN_QUALIFIED_NAMES, String.class); + List domainQualifiedNames = vertex.getMultiValuedProperty(ATTR_DOMAIN_QUALIFIED_NAMES, String.class); authorizeDomainAccess(domainQualifiedNames, AtlasPrivilege.ENTITY_UPDATE); From e784d28ef22331d2d019c6a366e7560484c8dae8 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Fri, 10 May 2024 23:08:41 +0530 Subject: [PATCH 166/316] DG-1297 reverted the maven.yml back to master's version --- .github/workflows/maven.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index b7b18d56b98..0977cb36a25 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -21,10 +21,11 @@ name: Java CI with Maven on: push: branches: - - staging + - alpha - beta - development - master + - lineageondemand jobs: build: From 73611455778b03f6c50c9e3acacbd253c806d80a Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Sat, 11 May 2024 12:41:49 +0530 Subject: [PATCH 167/316] DG-1297 Fix Re-run migration issue --- .../graph/v2/DataMeshQNMigrationService.java | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java index 862c316a831..500370d96d2 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java @@ -53,7 +53,6 @@ public class DataMeshQNMigrationService implements MigrationService, Runnable { private int counter; private final TransactionInterceptHelper transactionInterceptHelper; - private final RedisService redisService; @Inject public DataMeshQNMigrationService(AtlasEntityStore entityStore, RedisService redisService, EntityDiscoveryService discovery, EntityGraphRetriever entityRetriever, AtlasTypeRegistry typeRegistry, TransactionInterceptHelper transactionInterceptHelper) { @@ -64,11 +63,9 @@ public DataMeshQNMigrationService(AtlasEntityStore entityStore, RedisService red this.updatedPolicyResources = new HashMap<>(); this.counter = 0; this.transactionInterceptHelper = transactionInterceptHelper; - this.redisService = redisService; } public Boolean startMigration() throws Exception{ - redisService.putValue(DATA_MESH_QN, IN_PROGRESS); Set attributes = new HashSet<>(Arrays.asList(SUPER_DOMAIN_QN_ATTR, PARENT_DOMAIN_QN_ATTR, "__customAttributes")); @@ -79,13 +76,6 @@ public Boolean startMigration() throws Exception{ updateChunk(superDomain); } - if(errorOccured) { - redisService.putValue(DATA_MESH_QN, FAILED); - } else { - redisService.putValue(DATA_MESH_QN, SUCCESSFUL); - } - - LOG.info("MIGRATION_RESULT {} ",redisService.getValue(DATA_MESH_QN)); return Boolean.TRUE; } @@ -123,6 +113,12 @@ private void migrateDomainAttributes(AtlasVertex vertex, String parentDomainQual if(customAttributes != null && customAttributes.get(MIGRATION_CUSTOM_ATTRIBUTE) != null && customAttributes.get(MIGRATION_CUSTOM_ATTRIBUTE).equals("true")){ LOG.info("Entity already migrated for entity: {}", currentQualifiedName); + updatedQualifiedName = vertex.getProperty(QUALIFIED_NAME,String.class); + + if (StringUtils.isEmpty(superDomainQualifiedName)) { + superDomainQualifiedName = vertex.getProperty(QUALIFIED_NAME,String.class); + } + } else { superDomainQualifiedName = commitChangesInMemory(currentQualifiedName, updatedQualifiedName, parentDomainQualifiedName, superDomainQualifiedName, vertex, updatedAttributes); } From 0cdfcc3812948dada9b532e541c0cc306c95fefc Mon Sep 17 00:00:00 2001 From: hr2904 Date: Sat, 11 May 2024 16:03:28 +0530 Subject: [PATCH 168/316] Added forceRegen Query Param --- .../graph/v2/DataMeshQNMigrationService.java | 10 +++---- .../apache/atlas/web/rest/MigrationREST.java | 29 +++++++++++++------ 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java index 500370d96d2..88ef4bbd4cf 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java @@ -51,11 +51,10 @@ public class DataMeshQNMigrationService implements MigrationService, Runnable { boolean skipSuperDomain = false; private int counter; - + private boolean forceRegen; private final TransactionInterceptHelper transactionInterceptHelper; - @Inject - public DataMeshQNMigrationService(AtlasEntityStore entityStore, RedisService redisService, EntityDiscoveryService discovery, EntityGraphRetriever entityRetriever, AtlasTypeRegistry typeRegistry, TransactionInterceptHelper transactionInterceptHelper) { + public DataMeshQNMigrationService(AtlasEntityStore entityStore, EntityDiscoveryService discovery, EntityGraphRetriever entityRetriever, AtlasTypeRegistry typeRegistry, TransactionInterceptHelper transactionInterceptHelper,boolean forceRegen) { this.entityRetriever = entityRetriever; this.entityStore = entityStore; this.discovery = discovery; @@ -63,6 +62,7 @@ public DataMeshQNMigrationService(AtlasEntityStore entityStore, RedisService red this.updatedPolicyResources = new HashMap<>(); this.counter = 0; this.transactionInterceptHelper = transactionInterceptHelper; + this.forceRegen = forceRegen; } public Boolean startMigration() throws Exception{ @@ -110,7 +110,7 @@ private void migrateDomainAttributes(AtlasVertex vertex, String parentDomainQual Map updatedAttributes = new HashMap<>(); Map customAttributes = GraphHelper.getCustomAttributes(vertex); - if(customAttributes != null && customAttributes.get(MIGRATION_CUSTOM_ATTRIBUTE) != null && customAttributes.get(MIGRATION_CUSTOM_ATTRIBUTE).equals("true")){ + if(!this.forceRegen && customAttributes != null && customAttributes.get(MIGRATION_CUSTOM_ATTRIBUTE) != null && customAttributes.get(MIGRATION_CUSTOM_ATTRIBUTE).equals("true")){ LOG.info("Entity already migrated for entity: {}", currentQualifiedName); updatedQualifiedName = vertex.getProperty(QUALIFIED_NAME,String.class); @@ -231,7 +231,7 @@ private void migrateDataProductAttributes(AtlasVertex vertex, String parentDomai LOG.info("Migrating qualified name for Product: {}", currentQualifiedName); Map customAttributes = GraphHelper.getCustomAttributes(vertex); - if(customAttributes != null && customAttributes.get(MIGRATION_CUSTOM_ATTRIBUTE) != null && customAttributes.get(MIGRATION_CUSTOM_ATTRIBUTE).equals("true")) { + if(!this.forceRegen && customAttributes != null && customAttributes.get(MIGRATION_CUSTOM_ATTRIBUTE) != null && customAttributes.get(MIGRATION_CUSTOM_ATTRIBUTE).equals("true")) { LOG.info("Product already migrated: {}", currentQualifiedName); } else { diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java index d2281add5bc..1e57ec2f5b8 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java @@ -3,6 +3,7 @@ import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.RequestContext; import org.apache.atlas.annotation.Timed; +import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.instance.AtlasEntity; @@ -10,13 +11,11 @@ import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graphdb.*; import org.apache.atlas.repository.store.graph.AtlasEntityStore; -import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream; -import org.apache.atlas.repository.store.graph.v2.DataMeshQNMigrationService; -import org.apache.atlas.repository.store.graph.v2.EntityStream; -import org.apache.atlas.repository.store.graph.v2.MigrationService; +import org.apache.atlas.repository.store.graph.v2.*; import org.apache.atlas.repository.store.users.KeycloakStore; import org.apache.atlas.service.redis.RedisService; import org.apache.atlas.transformer.PreProcessorPoliciesTransformer; +import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfTracer; import org.apache.atlas.v1.model.instance.Id; import org.apache.atlas.web.util.Servlets; @@ -59,30 +58,42 @@ public class MigrationREST { private KeycloakStore keycloakStore; private AtlasGraph graph; DataMeshQNMigrationService dataMeshQNMigrationService; + + private final EntityGraphRetriever entityRetriever; private final RedisService redisService; + protected final AtlasTypeRegistry typeRegistry; + private final EntityDiscoveryService discovery; + + private final TransactionInterceptHelper transactionInterceptHelper; @Inject - public MigrationREST(AtlasEntityStore entityStore, AtlasGraph graph, DataMeshQNMigrationService dataMeshQNMigrationService,RedisService redisService) { + public MigrationREST(AtlasEntityStore entityStore, AtlasGraph graph, RedisService redisService,EntityDiscoveryService discovery, EntityGraphRetriever entityRetriever, AtlasTypeRegistry typeRegistry, TransactionInterceptHelper transactionInterceptHelper) { this.entityStore = entityStore; this.graph = graph; this.transformer = new PreProcessorPoliciesTransformer(); keycloakStore = new KeycloakStore(); this.redisService = redisService; - this.dataMeshQNMigrationService = dataMeshQNMigrationService; - + this.discovery = discovery; + this.entityRetriever = entityRetriever; + this.typeRegistry = typeRegistry; + this.transactionInterceptHelper = transactionInterceptHelper; } @POST @Path("submit") @Timed - public Boolean submit (@QueryParam("migrationType") String migrationType) throws Exception { + public Boolean submit (@QueryParam("migrationType") String migrationType,@QueryParam("forceMigration") boolean forceMigration) throws Exception { AtlasPerfTracer perf = null; MigrationService migrationService; try { if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "MigrationREST.submit()"); } - if( (MIGRATION + migrationType).equals(DATA_MESH_QN) ){ + if( (MIGRATION + migrationType).equals(DATA_MESH_QN)){ + if(Objects.isNull(forceMigration) ){ + forceMigration = false; + } + dataMeshQNMigrationService = new DataMeshQNMigrationService(entityStore, discovery, entityRetriever, typeRegistry, transactionInterceptHelper, forceMigration); dataMeshQNMigrationService.run(); } From 950ad04fa0620eb505d522c5c31a0a8c1c2c5f4b Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Sat, 11 May 2024 17:03:57 +0530 Subject: [PATCH 169/316] Fix bean dependency --- .../repository/store/graph/v2/DataMeshQNMigrationService.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java index 88ef4bbd4cf..8b244141c02 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java @@ -32,7 +32,6 @@ import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_CATEGORY; import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_RESOURCES; -@Component public class DataMeshQNMigrationService implements MigrationService, Runnable { private static final Logger LOG = LoggerFactory.getLogger(DataMeshQNMigrationService.class); @@ -54,7 +53,7 @@ public class DataMeshQNMigrationService implements MigrationService, Runnable { private boolean forceRegen; private final TransactionInterceptHelper transactionInterceptHelper; - public DataMeshQNMigrationService(AtlasEntityStore entityStore, EntityDiscoveryService discovery, EntityGraphRetriever entityRetriever, AtlasTypeRegistry typeRegistry, TransactionInterceptHelper transactionInterceptHelper,boolean forceRegen) { + public DataMeshQNMigrationService(AtlasEntityStore entityStore, EntityDiscoveryService discovery, EntityGraphRetriever entityRetriever, AtlasTypeRegistry typeRegistry, TransactionInterceptHelper transactionInterceptHelper, boolean forceRegen) { this.entityRetriever = entityRetriever; this.entityStore = entityStore; this.discovery = discovery; From 7af98dbb53feb0509fc4dab9a93474f55d0f40a4 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Sat, 11 May 2024 21:51:54 +0530 Subject: [PATCH 170/316] DG-1297 Handle Policy failure and skip --- .../graph/v2/DataMeshQNMigrationService.java | 21 +++++++++++-------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java index 8b244141c02..2a6391fd880 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java @@ -13,6 +13,7 @@ import org.apache.atlas.repository.store.graph.AtlasEntityStore; import org.apache.atlas.service.redis.RedisService; import org.apache.atlas.type.AtlasEntityType; +import org.apache.atlas.type.AtlasType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.util.NanoIdUtils; import org.apache.commons.collections.CollectionUtils; @@ -135,7 +136,7 @@ private void migrateDomainAttributes(AtlasVertex vertex, String parentDomainQual break; } catch (NullPointerException e){ - LOG.error("Null Pointer Exception occured for products with parent : {}",currentQualifiedName); + LOG.error("Null Pointer Exception occured for subdomains with parent : " + currentQualifiedName, e); } } @@ -152,7 +153,7 @@ private void migrateDomainAttributes(AtlasVertex vertex, String parentDomainQual break; } catch (NullPointerException e){ - LOG.error("Null Pointer Exception occured for subdomains with parent : {}",currentQualifiedName); + LOG.error("Null Pointer Exception occured for subdomains with parent : " + currentQualifiedName, e); } } @@ -166,21 +167,23 @@ private void migrateDomainAttributes(AtlasVertex vertex, String parentDomainQual public void commitChanges() { try { updatePolicy(this.updatedPolicyResources); - this.updatedPolicyResources.clear(); } catch (AtlasBaseException e) { this.errorOccured = true; this.skipSuperDomain = true; - throw new RuntimeException(e); + LOG.error("Failed to update set of policies: ", e); + LOG.error("Failed policies: {}", AtlasType.toJson(this.updatedPolicyResources)); + } finally { + this.updatedPolicyResources.clear(); } try { - if(!skipSuperDomain) { - transactionInterceptHelper.intercept(); - this.counter = 0; - } + transactionInterceptHelper.intercept(); } catch (Exception e){ this.skipSuperDomain = true; this.errorOccured = true; + LOG.error("Failed to commit set of assets: ", e); + } finally { + this.counter = 0; } } @@ -448,4 +451,4 @@ public void run() { throw new RuntimeException(e); } } -} \ No newline at end of file +} From e5a4e12829e59b60b46fdbb39accbea4e0300ddc Mon Sep 17 00:00:00 2001 From: akshaysw Date: Mon, 13 May 2024 11:32:09 +0530 Subject: [PATCH 171/316] add initial draft for daap visibility backend changes --- .../store/graph/v2/AtlasEntityStoreV2.java | 2 +- .../datamesh/DataProductPreProcessor.java | 113 +++++++++++++++++- 2 files changed, 110 insertions(+), 5 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 70fda21198b..85b3536d6cc 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -1808,7 +1808,7 @@ public PreProcessor getPreProcessor(String typeName) { break; case DATA_PRODUCT_ENTITY_TYPE: - preProcessor = new DataProductPreProcessor(typeRegistry, entityRetriever, graph); + preProcessor = new DataProductPreProcessor(typeRegistry, entityRetriever, graph, this); break; case QUERY_ENTITY_TYPE: diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index babebd7c902..31a65d19931 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -6,8 +6,12 @@ import org.apache.atlas.model.instance.*; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.AtlasEntityStore; +import org.apache.atlas.repository.store.graph.v2.AtlasEntityStream; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; +import org.apache.atlas.repository.store.graph.v2.EntityStream; +import org.apache.atlas.repository.util.AccessControlUtils; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; @@ -19,18 +23,21 @@ import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.*; +import static org.apache.atlas.repository.util.AccessControlUtils.*; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; public class DataProductPreProcessor extends AbstractDomainPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(DataProductPreProcessor.class); private EntityMutationContext context; + private AtlasEntityStore entityStore; private Map updatedPolicyResources; public DataProductPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, - AtlasGraph graph) { + AtlasGraph graph, AtlasEntityStore entityStore) { super(typeRegistry, entityRetriever, graph); this.updatedPolicyResources = new HashMap<>(); + this.entityStore = entityStore; } @Override @@ -44,18 +51,19 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co AtlasEntity entity = (AtlasEntity) entityStruct; + AtlasVertex vertex = context.getVertex(entity.getGuid()); + switch (operation) { case CREATE: - processCreateProduct(entity); + processCreateProduct(entity,vertex); break; case UPDATE: - AtlasVertex vertex = context.getVertex(entity.getGuid()); processUpdateProduct(entity, vertex); break; } } - private void processCreateProduct(AtlasEntity entity) throws AtlasBaseException { + private void processCreateProduct(AtlasEntity entity,AtlasVertex vertex) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreateProduct"); String productName = (String) entity.getAttribute(NAME); String parentDomainQualifiedName = (String) entity.getAttribute(PARENT_DOMAIN_QN_ATTR); @@ -70,6 +78,43 @@ private void processCreateProduct(AtlasEntity entity) throws AtlasBaseException productExists(productName, parentDomainQualifiedName); + String productGuid = vertex.getProperty("__guid", String.class); + + AtlasEntity policy = new AtlasEntity(); + policy.setTypeName(POLICY_ENTITY_TYPE); + policy.setAttribute(NAME,entity.getAttribute(NAME)); + policy.setAttribute(QUALIFIED_NAME, productGuid+"/read-policy"); + policy.setAttribute(AccessControlUtils.ATTR_POLICY_ACTIONS, Arrays.asList("entity-read")); + policy.setAttribute(ATTR_POLICY_CATEGORY,"datamesh"); + policy.setAttribute(ATTR_POLICY_TYPE,POLICY_TYPE_ALLOW); + policy.setAttribute(ATTR_POLICY_RESOURCES, Arrays.asList("entity:"+entity.getAttribute(QUALIFIED_NAME))); + policy.setAttribute("accessControlPolicyCategory",POLICY_CATEGORY_PERSONA); + policy.setAttribute("policyResourceCategory","entity"); + policy.setAttribute("policyServiceName","atlas"); + policy.setAttribute("policySubCategory","dataProduct"); + + switch ((String) entity.getAttribute("daapVisibility")) { + case "Private": + // do nothing for private daapVisibility + break; + case "Protected": + // create policy for policyUsers and policyGroups + policy.setAttribute(ATTR_POLICY_USERS, entity.getAttribute("daapVisibilityUsers")); + policy.setAttribute(ATTR_POLICY_GROUPS, entity.getAttribute("daapVisibilityGroups")); + break; + case "Public": + // set empty user list + policy.setAttribute(ATTR_POLICY_USERS, Arrays.asList()); + // set user groups oto public to allow access for all + policy.setAttribute(ATTR_POLICY_GROUPS, Arrays.asList("Public")); + break; + } + + AtlasEntity.AtlasEntitiesWithExtInfo policiesExtInfo = new AtlasEntity.AtlasEntitiesWithExtInfo(); + policiesExtInfo.addEntity(policy); + EntityStream entityStream = new AtlasEntityStream(policiesExtInfo); + entityStore.createOrUpdate(entityStream, false); // adding new policy + RequestContext.get().endMetricRecord(metricRecorder); } @@ -122,6 +167,66 @@ private void processUpdateProduct(AtlasEntity entity, AtlasVertex vertex) throws entity.setAttribute(QUALIFIED_NAME, vertexQnName); } + // check for daapVisibility change + String currentProductDaapVisibility = storedProduct.getAttribute("daapVisibility").toString(); + String newProductDaapVisibility = (String) entity.getAttribute(NAME); + + if (newProductDaapVisibility != null && !newProductDaapVisibility.equals(currentProductDaapVisibility)) { + + AtlasEntity policy = new AtlasEntity(); + policy.setTypeName(POLICY_ENTITY_TYPE); + policy.setAttribute(NAME,entity.getAttribute(NAME)); + policy.setAttribute(QUALIFIED_NAME,storedProduct.getGuid()+"/read-policy"); + + switch (currentProductDaapVisibility) { + case "Private": + switch (newProductDaapVisibility) { + case "Protected": + // create policy for policyUsers and policyGroups + policy.setAttribute(ATTR_POLICY_USERS, entity.getAttribute("daapVisibilityUsers")); + policy.setAttribute(ATTR_POLICY_GROUPS, entity.getAttribute("daapVisibilityGroups")); + break; + case "Public": + // set empty user list + policy.setAttribute(ATTR_POLICY_USERS, Arrays.asList()); + // set user groups oto public to allow access for all + policy.setAttribute(ATTR_POLICY_GROUPS, Arrays.asList("Public")); + } + break; + case "Protected": + switch (newProductDaapVisibility) { + case "Private": + // create policy for policyUsers and policyGroups + policy.setAttribute(ATTR_POLICY_USERS,Arrays.asList()); + policy.setAttribute(ATTR_POLICY_GROUPS,Arrays.asList()); + break; + case "Public": + // set empty user list + policy.setAttribute(ATTR_POLICY_USERS, Arrays.asList()); + // set user groups oto public to allow access for all + policy.setAttribute(ATTR_POLICY_GROUPS, Arrays.asList("Public")); + } + break; + case "Public": + switch (newProductDaapVisibility) { + case "Private": + // create policy for policyUsers and policyGroups + policy.setAttribute(ATTR_POLICY_USERS,Arrays.asList()); + policy.setAttribute(ATTR_POLICY_GROUPS,Arrays.asList()); + break; + case "Protected": + // create policy for policyUsers and policyGroups + policy.setAttribute(ATTR_POLICY_USERS, entity.getAttribute("daapVisibilityUsers")); + policy.setAttribute(ATTR_POLICY_GROUPS, entity.getAttribute("daapVisibilityGroups")); + break; + } + break; + } + AtlasEntity.AtlasEntitiesWithExtInfo policiesExtInfo = new AtlasEntity.AtlasEntitiesWithExtInfo(); + policiesExtInfo.addEntity(policy); + EntityStream entityStream = new AtlasEntityStream(policiesExtInfo); + entityStore.createOrUpdate(entityStream, false); // adding new policy + } RequestContext.get().endMetricRecord(metricRecorder); } From bdeb0c9ed85236dbd599cc949c9d75f7bf0dd346 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Mon, 13 May 2024 12:02:01 +0530 Subject: [PATCH 172/316] DG-1235 fix default titles not applicable issue --- .../accesscontrol/StakeholderPreProcessor.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java index 784f86d2f92..f3daf4636c8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java @@ -60,6 +60,7 @@ import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.indexSearchPaginated; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.verifyDuplicateAssetByName; import static org.apache.atlas.repository.store.graph.v2.preprocessor.datamesh.StakeholderTitlePreProcessor.ATTR_DOMAIN_QUALIFIED_NAMES; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.datamesh.StakeholderTitlePreProcessor.STAR; import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_ACCESS_CONTROL_ENABLED; import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_PERSONA_ROLE_ID; import static org.apache.atlas.repository.util.AccessControlUtils.REL_ATTR_POLICIES; @@ -259,8 +260,8 @@ protected void verifyDuplicateByDomainAndTitle(String domainQualifiedName, Strin List> mustClauseList = new ArrayList(); mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", STAKEHOLDER_ENTITY_TYPE))); mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); - mustClauseList.add(mapOf("term", mapOf("domainQualifiedName", domainQualifiedName))); - mustClauseList.add(mapOf("term", mapOf("stakeholderTitleGuid", stakeholderTitleGuid))); + mustClauseList.add(mapOf("term", mapOf(ATTR_DOMAIN_QUALIFIED_NAME, domainQualifiedName))); + mustClauseList.add(mapOf("term", mapOf(ATTR_STAKEHOLDER_TITLE_GUID, stakeholderTitleGuid))); Map bool = mapOf("must", mustClauseList); @@ -299,11 +300,13 @@ protected void ensureTitleAvailableForDomain(String domainQualifiedName, String List domainQualifiedNames = (List) stakeholderTitleHeader.getAttribute(ATTR_DOMAIN_QUALIFIED_NAMES); - Optional parentDomain = domainQualifiedNames.stream().filter(x -> domainQualifiedName.startsWith(x)).findFirst(); + if (!domainQualifiedNames.contains(STAR)) { + Optional parentDomain = domainQualifiedNames.stream().filter(x -> domainQualifiedName.startsWith(x)).findFirst(); - if (!parentDomain.isPresent()) { - throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, - String.format("Provided StakeholderTitle is not applicable to the domain")); + if (!parentDomain.isPresent()) { + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, + String.format("Provided StakeholderTitle is not applicable to the domain")); + } } } } From 590675dc57cb54dfcb8afeb1803c98702f1e53a4 Mon Sep 17 00:00:00 2001 From: akshaysw Date: Mon, 13 May 2024 12:04:14 +0530 Subject: [PATCH 173/316] use constants --- .../v2/preprocessor/datamesh/DataProductPreProcessor.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 31a65d19931..782ce15917d 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -89,9 +89,9 @@ private void processCreateProduct(AtlasEntity entity,AtlasVertex vertex) throws policy.setAttribute(ATTR_POLICY_TYPE,POLICY_TYPE_ALLOW); policy.setAttribute(ATTR_POLICY_RESOURCES, Arrays.asList("entity:"+entity.getAttribute(QUALIFIED_NAME))); policy.setAttribute("accessControlPolicyCategory",POLICY_CATEGORY_PERSONA); - policy.setAttribute("policyResourceCategory","entity"); - policy.setAttribute("policyServiceName","atlas"); - policy.setAttribute("policySubCategory","dataProduct"); + policy.setAttribute(ATTR_POLICY_RESOURCES_CATEGORY,"entity"); + policy.setAttribute(ATTR_POLICY_SERVICE_NAME,"atlas"); + policy.setAttribute(ATTR_POLICY_SUB_CATEGORY,"dataProduct"); switch ((String) entity.getAttribute("daapVisibility")) { case "Private": From caf4606da7168ace75836000fb4f427d40df7f15 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Mon, 13 May 2024 14:24:52 +0530 Subject: [PATCH 174/316] feat: fix few things like value settings and path mod --- .../graph/v2/ClassificationAssociator.java | 13 ---------- .../store/graph/v2/EntityGraphMapper.java | 24 +++++++++++++------ .../org/apache/atlas/web/rest/EntityREST.java | 7 +++--- 3 files changed, 20 insertions(+), 24 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java index 0c45042b28a..6cd9ac4e4b8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/ClassificationAssociator.java @@ -154,19 +154,6 @@ public Updater(AtlasTypeRegistry typeRegistry, AtlasEntityStore entitiesStore, this(AtlasGraphProvider.getGraphInstance(), typeRegistry, entitiesStore, entityGraphMapper, entityChangeNotifier, instanceConverter); } - public void repairClassificationsMappings(List guids) throws AtlasBaseException { - - for (String guid : guids) { - // Fetch attached classifications - AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(this.graph, guid); - List classifications = entityRetriever.getAllClassifications(vertex); - for (AtlasClassification classification : classifications) { - // Fetch the entity header - - } - } - } - public void setClassifications(Map map) throws AtlasBaseException { RequestContext.get().setDelayTagNotifications(true); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 586bdd757c4..52d5668b447 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -2991,7 +2991,7 @@ public void repairClassificationMappings(AtlasEntityHeader entityHeader, AtlasVe for (AtlasClassification classification : classifications) { String typeName = classification.getTypeName(); classificationText = fullTextMapperV2.getClassificationTextForEntity(new AtlasEntity(entityHeader)); - if(classification.isPropagate()) { + if(classification.isPropagate() && !classification.getEntityGuid().equals(entityHeader.getGuid())) { //Add to propagatedClassificationNamesList propagatedClassificationNamesList.add(typeName); //Create delimiter separated string for propagatedClassificationNames @@ -3011,15 +3011,25 @@ public void repairClassificationMappings(AtlasEntityHeader entityHeader, AtlasVe } } } + //Delete all the properties + entityVertex.removeProperty(TRAIT_NAMES_PROPERTY_KEY); + entityVertex.removeProperty(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY); - //Update classificationNames and propagatedClassificationNames in entityVertex - AtlasGraphUtilsV2.setEncodedProperty(entityVertex, CLASSIFICATION_NAMES_KEY, classificationNames); - AtlasGraphUtilsV2.setEncodedProperty(entityVertex, CLASSIFICATION_TEXT_KEY, classificationText); - AtlasGraphUtilsV2.setEncodedProperty(entityVertex, PROPAGATED_CLASSIFICATION_NAMES_KEY, propagatedClassificationName); + //Update classificationNames and propagatedClassificationNames in entityVertex + AtlasGraphUtilsV2.setProperty(entityVertex, CLASSIFICATION_NAMES_KEY, classificationNames); + AtlasGraphUtilsV2.setProperty(entityVertex, CLASSIFICATION_TEXT_KEY, classificationText); + AtlasGraphUtilsV2.setProperty(entityVertex, PROPAGATED_CLASSIFICATION_NAMES_KEY, propagatedClassificationName); + // Make classificationNames unique list as it is of type SET + classificationNamesList = classificationNamesList.stream().distinct().collect(Collectors.toList()); //Update classificationNames and propagatedClassificationNames in entityHeader - AtlasGraphUtilsV2.setEncodedProperty(entityVertex, TRAIT_NAMES_PROPERTY_KEY, classificationNamesList); - AtlasGraphUtilsV2.setEncodedProperty(entityVertex, PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, propagatedClassificationNamesList); + for(String classificationName : classificationNamesList) { + AtlasGraphUtilsV2.addEncodedProperty(entityVertex, TRAIT_NAMES_PROPERTY_KEY, classificationName); + } + for (String classificationName : propagatedClassificationNamesList) { + //AtlasGraphUtilsV2.addProperty(entityVertex, PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationName); + entityVertex.addListProperty(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationName); + } } } diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java index 6b3b1b52b37..0ebb48b16c6 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java @@ -1275,11 +1275,11 @@ public void setClassifications(AtlasEntityHeaders entityHeaders) throws AtlasBas } @POST - @Path("bulk/repairClassificationsMappings") + @Path("repairClassificationsMappings/{guid}") @Produces(Servlets.JSON_MEDIA_TYPE) @Consumes(Servlets.JSON_MEDIA_TYPE) @Timed - public void repairClassifications(List guids) throws AtlasBaseException { + public void repairClassifications(@PathParam("guid") String guid) throws AtlasBaseException { AtlasPerfTracer perf = null; try { @@ -1287,8 +1287,7 @@ public void repairClassifications(List guids) throws AtlasBaseException perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityREST.repairClassifications()"); } - ClassificationAssociator.Updater associator = new ClassificationAssociator.Updater(typeRegistry, entitiesStore, entityGraphMapper, entityChangeNotifier, instanceConverter); - associator.repairClassificationsMappings(guids); + entitiesStore.repairClassificationMappings(guid); } finally { AtlasPerfTracer.log(perf); } From b2c805e9b2acdd4bbb134925ebc42fb7d6376b89 Mon Sep 17 00:00:00 2001 From: akshaysw Date: Mon, 13 May 2024 15:08:19 +0530 Subject: [PATCH 175/316] add refactoring changes --- .../v2/preprocessor/PreProcessorUtils.java | 4 +- .../datamesh/DataProductPreProcessor.java | 213 ++++++++++-------- 2 files changed, 121 insertions(+), 96 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index 9639cdd3179..eeb3196b4c3 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -53,7 +53,9 @@ public class PreProcessorUtils { public static final String PARENT_DOMAIN_QN_ATTR = "parentDomainQualifiedName"; public static final String SUPER_DOMAIN_QN_ATTR = "superDomainQualifiedName"; - + public static final String DAAP_VISIBILITY = "daapVisibility"; + public static final String DAAP_VISIBILITY_USERS = "daapVisibilityUsers"; + public static final String DAAP_VISIBILITY_GROUPS = "daapVisibilityGroups"; //Migration Constants public static final String MIGRATION = "MIGRATION:"; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 782ce15917d..252e1f56d08 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -11,7 +11,7 @@ import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; import org.apache.atlas.repository.store.graph.v2.EntityStream; -import org.apache.atlas.repository.util.AccessControlUtils; +import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; @@ -24,10 +24,14 @@ import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.*; import static org.apache.atlas.repository.util.AccessControlUtils.*; -import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; public class DataProductPreProcessor extends AbstractDomainPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(DataProductPreProcessor.class); + private static final String PRIVATE = "Private"; + private static final String PROTECTED = "Protected"; + private static final String PUBLIC = "Public"; + + private EntityMutationContext context; private AtlasEntityStore entityStore; @@ -78,42 +82,7 @@ private void processCreateProduct(AtlasEntity entity,AtlasVertex vertex) throws productExists(productName, parentDomainQualifiedName); - String productGuid = vertex.getProperty("__guid", String.class); - - AtlasEntity policy = new AtlasEntity(); - policy.setTypeName(POLICY_ENTITY_TYPE); - policy.setAttribute(NAME,entity.getAttribute(NAME)); - policy.setAttribute(QUALIFIED_NAME, productGuid+"/read-policy"); - policy.setAttribute(AccessControlUtils.ATTR_POLICY_ACTIONS, Arrays.asList("entity-read")); - policy.setAttribute(ATTR_POLICY_CATEGORY,"datamesh"); - policy.setAttribute(ATTR_POLICY_TYPE,POLICY_TYPE_ALLOW); - policy.setAttribute(ATTR_POLICY_RESOURCES, Arrays.asList("entity:"+entity.getAttribute(QUALIFIED_NAME))); - policy.setAttribute("accessControlPolicyCategory",POLICY_CATEGORY_PERSONA); - policy.setAttribute(ATTR_POLICY_RESOURCES_CATEGORY,"entity"); - policy.setAttribute(ATTR_POLICY_SERVICE_NAME,"atlas"); - policy.setAttribute(ATTR_POLICY_SUB_CATEGORY,"dataProduct"); - - switch ((String) entity.getAttribute("daapVisibility")) { - case "Private": - // do nothing for private daapVisibility - break; - case "Protected": - // create policy for policyUsers and policyGroups - policy.setAttribute(ATTR_POLICY_USERS, entity.getAttribute("daapVisibilityUsers")); - policy.setAttribute(ATTR_POLICY_GROUPS, entity.getAttribute("daapVisibilityGroups")); - break; - case "Public": - // set empty user list - policy.setAttribute(ATTR_POLICY_USERS, Arrays.asList()); - // set user groups oto public to allow access for all - policy.setAttribute(ATTR_POLICY_GROUPS, Arrays.asList("Public")); - break; - } - - AtlasEntity.AtlasEntitiesWithExtInfo policiesExtInfo = new AtlasEntity.AtlasEntitiesWithExtInfo(); - policiesExtInfo.addEntity(policy); - EntityStream entityStream = new AtlasEntityStream(policiesExtInfo); - entityStore.createOrUpdate(entityStream, false); // adding new policy + createDaapVisibilityPolicy(entity,vertex); RequestContext.get().endMetricRecord(metricRecorder); } @@ -168,65 +137,13 @@ private void processUpdateProduct(AtlasEntity entity, AtlasVertex vertex) throws } // check for daapVisibility change - String currentProductDaapVisibility = storedProduct.getAttribute("daapVisibility").toString(); - String newProductDaapVisibility = (String) entity.getAttribute(NAME); + String currentProductDaapVisibility = storedProduct.getAttribute(DAAP_VISIBILITY).toString(); + String newProductDaapVisibility = (String) entity.getAttribute(DAAP_VISIBILITY); if (newProductDaapVisibility != null && !newProductDaapVisibility.equals(currentProductDaapVisibility)) { - - AtlasEntity policy = new AtlasEntity(); - policy.setTypeName(POLICY_ENTITY_TYPE); - policy.setAttribute(NAME,entity.getAttribute(NAME)); - policy.setAttribute(QUALIFIED_NAME,storedProduct.getGuid()+"/read-policy"); - - switch (currentProductDaapVisibility) { - case "Private": - switch (newProductDaapVisibility) { - case "Protected": - // create policy for policyUsers and policyGroups - policy.setAttribute(ATTR_POLICY_USERS, entity.getAttribute("daapVisibilityUsers")); - policy.setAttribute(ATTR_POLICY_GROUPS, entity.getAttribute("daapVisibilityGroups")); - break; - case "Public": - // set empty user list - policy.setAttribute(ATTR_POLICY_USERS, Arrays.asList()); - // set user groups oto public to allow access for all - policy.setAttribute(ATTR_POLICY_GROUPS, Arrays.asList("Public")); - } - break; - case "Protected": - switch (newProductDaapVisibility) { - case "Private": - // create policy for policyUsers and policyGroups - policy.setAttribute(ATTR_POLICY_USERS,Arrays.asList()); - policy.setAttribute(ATTR_POLICY_GROUPS,Arrays.asList()); - break; - case "Public": - // set empty user list - policy.setAttribute(ATTR_POLICY_USERS, Arrays.asList()); - // set user groups oto public to allow access for all - policy.setAttribute(ATTR_POLICY_GROUPS, Arrays.asList("Public")); - } - break; - case "Public": - switch (newProductDaapVisibility) { - case "Private": - // create policy for policyUsers and policyGroups - policy.setAttribute(ATTR_POLICY_USERS,Arrays.asList()); - policy.setAttribute(ATTR_POLICY_GROUPS,Arrays.asList()); - break; - case "Protected": - // create policy for policyUsers and policyGroups - policy.setAttribute(ATTR_POLICY_USERS, entity.getAttribute("daapVisibilityUsers")); - policy.setAttribute(ATTR_POLICY_GROUPS, entity.getAttribute("daapVisibilityGroups")); - break; - } - break; - } - AtlasEntity.AtlasEntitiesWithExtInfo policiesExtInfo = new AtlasEntity.AtlasEntitiesWithExtInfo(); - policiesExtInfo.addEntity(policy); - EntityStream entityStream = new AtlasEntityStream(policiesExtInfo); - entityStore.createOrUpdate(entityStream, false); // adding new policy + updateDaapVisibilityPolicy(entity, storedProduct, currentProductDaapVisibility,newProductDaapVisibility); } + RequestContext.get().endMetricRecord(metricRecorder); } @@ -291,7 +208,113 @@ private static String createQualifiedName(String parentDomainQualifiedName) thro if (StringUtils.isEmpty(parentDomainQualifiedName)) { throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Parent Domain Qualified Name cannot be empty or null"); } - return parentDomainQualifiedName + "/product/" + getUUID(); + return parentDomainQualifiedName + "/product/" + PreProcessorUtils.getUUID(); + + } + + private void createDaapVisibilityPolicy(AtlasEntity entity,AtlasVertex vertex) throws AtlasBaseException { + String productGuid = vertex.getProperty("__guid", String.class); + AtlasEntity policy = new AtlasEntity(); + policy.setTypeName(POLICY_ENTITY_TYPE); + policy.setAttribute(NAME, entity.getAttribute(NAME)); + policy.setAttribute(QUALIFIED_NAME, productGuid + "/read-policy"); + policy.setAttribute(ATTR_POLICY_ACTIONS, Arrays.asList("entity-read")); + policy.setAttribute(ATTR_POLICY_CATEGORY, MESH_POLICY_CATEGORY); + policy.setAttribute(ATTR_POLICY_TYPE, POLICY_TYPE_ALLOW); + policy.setAttribute(ATTR_POLICY_RESOURCES, Arrays.asList("entity:" + entity.getAttribute(QUALIFIED_NAME))); + policy.setAttribute(ATTR_POLICY_RESOURCES_CATEGORY, POLICY_RESOURCE_CATEGORY_PERSONA_ENTITY); + policy.setAttribute(ATTR_POLICY_SERVICE_NAME, "atlas"); + policy.setAttribute(ATTR_POLICY_SUB_CATEGORY, DATA_PRODUCT_ENTITY_TYPE); + + switch ((String) entity.getAttribute(DAAP_VISIBILITY)) { + case PRIVATE: + // do nothing for private daapVisibility + break; + case PROTECTED: + // create policy for policyUsers and policyGroups + policy.setAttribute(ATTR_POLICY_USERS, entity.getAttribute(DAAP_VISIBILITY_USERS)); + policy.setAttribute(ATTR_POLICY_GROUPS, entity.getAttribute(DAAP_VISIBILITY_GROUPS)); + break; + case PUBLIC: + // set empty user list + policy.setAttribute(ATTR_POLICY_USERS, Arrays.asList()); + // set user groups oto public to allow access for all + policy.setAttribute(ATTR_POLICY_GROUPS, Arrays.asList("public")); + break; + } + + try { + RequestContext.get().setSkipAuthorizationCheck(true); + AtlasEntity.AtlasEntitiesWithExtInfo policiesExtInfo = new AtlasEntity.AtlasEntitiesWithExtInfo(); + policiesExtInfo.addEntity(policy); + EntityStream entityStream = new AtlasEntityStream(policiesExtInfo); + entityStore.createOrUpdate(entityStream, false); // adding new policy + } finally { + RequestContext.get().setSkipAuthorizationCheck(false); + } + } + + private void updateDaapVisibilityPolicy(AtlasEntity newEntity, AtlasEntity currentEntity, String currentProductDaapVisibility, String newProductDaapVisibility) throws AtlasBaseException{ + + AtlasEntity policy = new AtlasEntity(); + policy.setTypeName(POLICY_ENTITY_TYPE); + policy.setAttribute(NAME,newEntity.getAttribute(NAME)); + policy.setAttribute(QUALIFIED_NAME,currentEntity.getGuid()+"/read-policy"); + + switch (currentProductDaapVisibility) { + case PRIVATE: + switch (newProductDaapVisibility) { + case PROTECTED: + // create policy for policyUsers and policyGroups + policy.setAttribute(ATTR_POLICY_USERS, newEntity.getAttribute(DAAP_VISIBILITY_USERS)); + policy.setAttribute(ATTR_POLICY_GROUPS, newEntity.getAttribute(DAAP_VISIBILITY_GROUPS)); + break; + case PUBLIC: + // set empty user list + policy.setAttribute(ATTR_POLICY_USERS, Arrays.asList()); + // set user groups to public to allow access for all + policy.setAttribute(ATTR_POLICY_GROUPS, Arrays.asList("public")); + } + break; + case PROTECTED: + switch (newProductDaapVisibility) { + case PRIVATE: + // create policy for policyUsers and policyGroups + policy.setAttribute(ATTR_POLICY_USERS,Arrays.asList()); + policy.setAttribute(ATTR_POLICY_GROUPS,Arrays.asList()); + break; + case PUBLIC: + // set empty user list + policy.setAttribute(ATTR_POLICY_USERS, Arrays.asList()); + // set user groups to public to allow access for all + policy.setAttribute(ATTR_POLICY_GROUPS, Arrays.asList("public")); + } + break; + case PUBLIC: + switch (newProductDaapVisibility) { + case PRIVATE: + // create policy for policyUsers and policyGroups + policy.setAttribute(ATTR_POLICY_USERS,Arrays.asList()); + policy.setAttribute(ATTR_POLICY_GROUPS,Arrays.asList()); + break; + case PROTECTED: + // create policy for policyUsers and policyGroups + policy.setAttribute(ATTR_POLICY_USERS, newEntity.getAttribute(DAAP_VISIBILITY_USERS)); + policy.setAttribute(ATTR_POLICY_GROUPS, newEntity.getAttribute(DAAP_VISIBILITY_GROUPS)); + break; + } + break; + } + + try { + RequestContext.get().setSkipAuthorizationCheck(true); + AtlasEntity.AtlasEntitiesWithExtInfo policiesExtInfo = new AtlasEntity.AtlasEntitiesWithExtInfo(); + policiesExtInfo.addEntity(policy); + EntityStream entityStream = new AtlasEntityStream(policiesExtInfo); + entityStore.createOrUpdate(entityStream, false); // adding new policy + } finally { + RequestContext.get().setSkipAuthorizationCheck(false); + } } } From ff996b9105c940b1212bb15d6230b098faa29e5a Mon Sep 17 00:00:00 2001 From: akshaysw Date: Mon, 13 May 2024 15:14:33 +0530 Subject: [PATCH 176/316] add daapvisibility check in move product logic --- .../datamesh/DataProductPreProcessor.java | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 252e1f56d08..b9278ef5a6d 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -113,7 +113,16 @@ private void processUpdateProduct(AtlasEntity entity, AtlasVertex vertex) throws newParentDomainQualifiedName = (String) newParentDomainHeader.getAttribute(QUALIFIED_NAME); } + // check for daapVisibility change + String currentProductDaapVisibility = storedProduct.getAttribute(DAAP_VISIBILITY).toString(); + String newProductDaapVisibility = (String) entity.getAttribute(DAAP_VISIBILITY); + if (newParentDomainQualifiedName != null && !newParentDomainQualifiedName.equals(currentParentDomainQualifiedName)) { + + if (newProductDaapVisibility != null && !newProductDaapVisibility.equals(currentProductDaapVisibility)){ + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Moving the product to another domain, along with the change in Dapp visibility, is not allowed"); + } + //Auth check isAuthorized(currentParentDomainHeader, newParentDomainHeader); @@ -136,10 +145,6 @@ private void processUpdateProduct(AtlasEntity entity, AtlasVertex vertex) throws entity.setAttribute(QUALIFIED_NAME, vertexQnName); } - // check for daapVisibility change - String currentProductDaapVisibility = storedProduct.getAttribute(DAAP_VISIBILITY).toString(); - String newProductDaapVisibility = (String) entity.getAttribute(DAAP_VISIBILITY); - if (newProductDaapVisibility != null && !newProductDaapVisibility.equals(currentProductDaapVisibility)) { updateDaapVisibilityPolicy(entity, storedProduct, currentProductDaapVisibility,newProductDaapVisibility); } From 1b487152ab41cbb4549f0b0bfa15178e191ef016 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Mon, 13 May 2024 15:19:42 +0530 Subject: [PATCH 177/316] DG-1235 Fix NPE and code refactorings --- .../store/graph/v2/AtlasEntityStoreV2.java | 2 +- .../v2/preprocessor/PreProcessorUtils.java | 5 +- .../StakeholderPreProcessor.java | 80 +++++++++-------- .../datamesh/DataDomainPreProcessor.java | 9 +- .../StakeholderTitlePreProcessor.java | 90 ++++++++++--------- 5 files changed, 102 insertions(+), 84 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 70b76120d00..bdb088d48b6 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -1858,7 +1858,7 @@ public PreProcessor getPreProcessor(String typeName) { break; case "StakeholderTitle": - preProcessor = new StakeholderTitlePreProcessor(graph, typeRegistry, entityRetriever, this); + preProcessor = new StakeholderTitlePreProcessor(graph, typeRegistry, entityRetriever); break; } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index ba449f53ed9..40ebc715c3c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -22,11 +22,9 @@ import java.util.*; -import static org.apache.atlas.repository.Constants.NAME; import static org.apache.atlas.repository.Constants.QUERY_COLLECTION_ENTITY_TYPE; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY; -import static org.apache.atlas.repository.Constants.STAKEHOLDER_ENTITY_TYPE; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; public class PreProcessorUtils { @@ -48,6 +46,7 @@ public class PreProcessorUtils { public static final String DATA_PRODUCT_REL_TYPE = "dataProducts"; public static final String MIGRATION_CUSTOM_ATTRIBUTE = "isQualifiedNameMigrated"; public static final String DATA_DOMAIN_REL_TYPE = "dataDomain"; + public static final String STAKEHOLDER_REL_TYPE = "stakeholders"; public static final String MESH_POLICY_CATEGORY = "datamesh"; @@ -179,8 +178,6 @@ public static List indexSearchPaginated(Map d } public static void verifyDuplicateAssetByName(String typeName, String assetName, EntityDiscoveryService discovery, String errorMessage) throws AtlasBaseException { - boolean exists = false; - List> mustClauseList = new ArrayList(); mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", typeName))); mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java index f3daf4636c8..a8c01effbfe 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java @@ -31,7 +31,6 @@ import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.instance.AtlasStruct; import org.apache.atlas.model.instance.EntityMutations; -import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.AtlasEntityStore; @@ -40,17 +39,18 @@ import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; +import static java.lang.String.format; import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; import static org.apache.atlas.AtlasErrorCode.OPERATION_NOT_SUPPORTED; import static org.apache.atlas.repository.Constants.NAME; @@ -58,7 +58,6 @@ import static org.apache.atlas.repository.Constants.STAKEHOLDER_ENTITY_TYPE; import static org.apache.atlas.repository.Constants.STAKEHOLDER_TITLE_ENTITY_TYPE; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.indexSearchPaginated; -import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.verifyDuplicateAssetByName; import static org.apache.atlas.repository.store.graph.v2.preprocessor.datamesh.StakeholderTitlePreProcessor.ATTR_DOMAIN_QUALIFIED_NAMES; import static org.apache.atlas.repository.store.graph.v2.preprocessor.datamesh.StakeholderTitlePreProcessor.STAR; import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_ACCESS_CONTROL_ENABLED; @@ -116,27 +115,26 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co @Override public void processDelete(AtlasVertex vertex) throws AtlasBaseException { AtlasEntity.AtlasEntityWithExtInfo entityWithExtInfo = entityRetriever.toAtlasEntityWithExtInfo(vertex); - AtlasEntity persona = entityWithExtInfo.getEntity(); + AtlasEntity stakeholder = entityWithExtInfo.getEntity(); - if(!persona.getStatus().equals(AtlasEntity.Status.ACTIVE)) { - LOG.info("Persona with guid {} is already deleted/purged", persona.getGuid()); + if(!stakeholder.getStatus().equals(AtlasEntity.Status.ACTIVE)) { + LOG.info("Stakeholder is already deleted/purged"); return; } //delete policies - List policies = (List) persona.getRelationshipAttribute(REL_ATTR_POLICIES); + List policies = (List) stakeholder.getRelationshipAttribute(REL_ATTR_POLICIES); if (CollectionUtils.isNotEmpty(policies)) { for (AtlasObjectId policyObjectId : policies) { - //AtlasVertex policyVertex = entityRetriever.getEntityVertex(policyObjectId.getGuid()); entityStore.deleteById(policyObjectId.getGuid()); } } //remove role - keycloakStore.removeRole(getPersonaRoleId(persona)); + keycloakStore.removeRole(getPersonaRoleId(stakeholder)); //delete ES alias - aliasStore.deleteAlias(getESAliasName(persona)); + aliasStore.deleteAlias(getESAliasName(stakeholder)); } private void processCreateStakeholder(AtlasEntity entity) throws AtlasBaseException { @@ -145,7 +143,7 @@ private void processCreateStakeholder(AtlasEntity entity) throws AtlasBaseExcept validateNoPoliciesAttached(entity); if (!entity.hasRelationshipAttribute(REL_ATTR_STAKEHOLDER_TITLE) || !entity.hasRelationshipAttribute(REL_ATTR_STAKEHOLDER_DOMAIN)) { - throw new AtlasBaseException(BAD_REQUEST, "Relations stakeholderTitle and dataDomain are mandatory"); + throw new AtlasBaseException(BAD_REQUEST, "Relationships stakeholderTitle and dataDomain are mandatory"); } String domainQualifiedName = getQualifiedNameFromRelationAttribute(entity, REL_ATTR_STAKEHOLDER_DOMAIN); @@ -154,21 +152,20 @@ private void processCreateStakeholder(AtlasEntity entity) throws AtlasBaseExcept ensureTitleAvailableForDomain(domainQualifiedName, stakeholderTitleGuid); //validate Stakeholder & StakeholderTitle pair is unique for this domain - verifyDuplicateByDomainAndTitle(domainQualifiedName, stakeholderTitleGuid); + verifyDuplicateStakeholderByDomainAndTitle(domainQualifiedName, stakeholderTitleGuid); //validate Name uniqueness for Stakeholders across this domain String name = (String) entity.getAttribute(NAME); - verifyDuplicateAssetByName(STAKEHOLDER_ENTITY_TYPE, name, discovery, - String.format("Stakeholder with name %s already exists for current domain", name)); + verifyDuplicateStakeholderByName(name, domainQualifiedName, discovery); entity.setAttribute(ATTR_DOMAIN_QUALIFIED_NAME, domainQualifiedName); entity.setAttribute(ATTR_STAKEHOLDER_TITLE_GUID, stakeholderTitleGuid); - String personaQualifiedName = String.format("default/%s/%s", + String stakeholderQualifiedName = format("default/%s/%s", getUUID(), domainQualifiedName); - entity.setAttribute(QUALIFIED_NAME, personaQualifiedName); + entity.setAttribute(QUALIFIED_NAME, stakeholderQualifiedName); AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, new AtlasEntityHeader(entity)), @@ -192,32 +189,32 @@ private void processUpdateStakeholder(EntityMutationContext context, AtlasEntity validateNoPoliciesAttached(stakeholder); - validateNoPoliciesAttached(stakeholder); AtlasVertex vertex = context.getVertex(stakeholder.getGuid()); - AtlasEntity existingStakeholderEntity = entityRetriever.toAtlasEntityWithExtInfo(vertex).getEntity(); + AtlasEntity existingStakeholderEntity = entityRetriever.toAtlasEntity(vertex); if (!AtlasEntity.Status.ACTIVE.equals(existingStakeholderEntity.getStatus())) { - throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Stakeholder not Active"); + throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Stakeholder is not Active"); } stakeholder.removeAttribute(ATTR_DOMAIN_QUALIFIED_NAME); stakeholder.removeAttribute(ATTR_STAKEHOLDER_TITLE_GUID); - stakeholder.getRelationshipAttributes().remove(REL_ATTR_STAKEHOLDER_DOMAIN); - stakeholder.getRelationshipAttributes().remove(REL_ATTR_STAKEHOLDER_TITLE); + stakeholder.removeAttribute(ATTR_PERSONA_ROLE_ID); + if (MapUtils.isNotEmpty(stakeholder.getRelationshipAttributes())) { + stakeholder.getRelationshipAttributes().remove(REL_ATTR_STAKEHOLDER_DOMAIN); + stakeholder.getRelationshipAttributes().remove(REL_ATTR_STAKEHOLDER_TITLE); + } String currentName = vertex.getProperty(NAME, String.class); String newName = (String) stakeholder.getAttribute(NAME); if (!currentName.equals(newName)) { - verifyDuplicateAssetByName(STAKEHOLDER_ENTITY_TYPE, newName, discovery, - String.format("Stakeholder with name %s already exists for current domain", newName)); + verifyDuplicateStakeholderByName(newName, (String) existingStakeholderEntity.getAttribute(ATTR_DOMAIN_QUALIFIED_NAME), discovery); } String vertexQName = vertex.getProperty(QUALIFIED_NAME, String.class); stakeholder.setAttribute(QUALIFIED_NAME, vertexQName); - stakeholder.setAttribute(ATTR_PERSONA_ROLE_ID, getPersonaRoleId(existingStakeholderEntity)); AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, new AtlasEntityHeader(stakeholder)), "update Stakeholder: ", stakeholder.getAttribute(NAME)); @@ -255,7 +252,7 @@ private String getQualifiedNameFromRelationAttribute(AtlasEntity entity, String return qualifiedName; } - protected void verifyDuplicateByDomainAndTitle(String domainQualifiedName, String stakeholderTitleGuid) throws AtlasBaseException { + protected void verifyDuplicateStakeholderByDomainAndTitle(String domainQualifiedName, String stakeholderTitleGuid) throws AtlasBaseException { List> mustClauseList = new ArrayList(); mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", STAKEHOLDER_ENTITY_TYPE))); @@ -265,14 +262,13 @@ protected void verifyDuplicateByDomainAndTitle(String domainQualifiedName, Strin Map bool = mapOf("must", mustClauseList); - Map dsl = mapOf("query", mapOf("bool", bool)); List assets = indexSearchPaginated(dsl, null, this.discovery); if (CollectionUtils.isNotEmpty(assets)) { throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, - String.format("Stakeholder for provided title & domain combination already exists")); + format("Stakeholder for provided title & domain combination already exists with name: {}", assets.get(0).getAttribute(NAME))); } } @@ -283,14 +279,7 @@ protected void ensureTitleAvailableForDomain(String domainQualifiedName, String mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); mustClauseList.add(mapOf("term", mapOf("__guid", stakeholderTitleGuid))); - /*List> shouldClauseList = new ArrayList(); - shouldClauseList.add(mapOf("term", mapOf("domainQualifiedNames", domainQualifiedName))); - shouldClauseList.add(mapOf("term", mapOf("domainQualifiedNames", "*"))); - mustClauseList.add(mapOf("bool", mapOf("should", shouldClauseList))); - */ - Map bool = mapOf("must", mustClauseList); - Map dsl = mapOf("query", mapOf("bool", bool)); List assets = indexSearchPaginated(dsl, Collections.singleton(ATTR_DOMAIN_QUALIFIED_NAMES), this.discovery); @@ -304,10 +293,29 @@ protected void ensureTitleAvailableForDomain(String domainQualifiedName, String Optional parentDomain = domainQualifiedNames.stream().filter(x -> domainQualifiedName.startsWith(x)).findFirst(); if (!parentDomain.isPresent()) { - throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, - String.format("Provided StakeholderTitle is not applicable to the domain")); + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Provided StakeholderTitle is not applicable to the domain"); } } } } + + public static void verifyDuplicateStakeholderByName(String assetName, String domainQualifiedName, EntityDiscoveryService discovery) throws AtlasBaseException { + + List> mustClauseList = new ArrayList(); + mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", STAKEHOLDER_ENTITY_TYPE))); + mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); + mustClauseList.add(mapOf("term", mapOf("name.keyword", assetName))); + mustClauseList.add(mapOf("term", mapOf(ATTR_DOMAIN_QUALIFIED_NAME, domainQualifiedName))); + + + Map bool = mapOf("must", mustClauseList); + Map dsl = mapOf("query", mapOf("bool", bool)); + + List assets = indexSearchPaginated(dsl, null, discovery); + + if (CollectionUtils.isNotEmpty(assets)) { + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, + format("Stakeholder with name %s already exists for current domain", assetName)); + } + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java index b02c39f19eb..db1a8cc1830 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java @@ -83,6 +83,11 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co private void processCreateDomain(AtlasEntity entity) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreateDomain"); + + if(entity.hasRelationshipAttribute(STAKEHOLDER_REL_TYPE)){ + throw new AtlasBaseException(AtlasErrorCode.OPERATION_NOT_SUPPORTED, "Managing Stakeholders while creating a domain"); + } + String domainName = (String) entity.getAttribute(NAME); String parentDomainQualifiedName = (String) entity.getAttribute(PARENT_DOMAIN_QN_ATTR); @@ -108,8 +113,8 @@ private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Cannot update Domain's subDomains or dataProducts relations"); } - if(entity.hasRelationshipAttribute("stakeholders")){ - throw new AtlasBaseException(AtlasErrorCode.OPERATION_NOT_SUPPORTED, "Managing Stakeholders via Domain update"); + if(entity.hasRelationshipAttribute(STAKEHOLDER_REL_TYPE)){ + throw new AtlasBaseException(AtlasErrorCode.OPERATION_NOT_SUPPORTED, "Managing Stakeholders while updating a domain"); } String vertexQnName = vertex.getProperty(QUALIFIED_NAME, String.class); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java index 185f6110499..7396cdde9eb 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java @@ -26,21 +26,17 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Collections; import java.util.List; -import java.util.Map; import java.util.Optional; +import static java.lang.String.format; import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; import static org.apache.atlas.AtlasErrorCode.OPERATION_NOT_SUPPORTED; import static org.apache.atlas.repository.Constants.DATA_DOMAIN_ENTITY_TYPE; import static org.apache.atlas.repository.Constants.NAME; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; -import static org.apache.atlas.repository.Constants.STAKEHOLDER_ENTITY_TYPE; import static org.apache.atlas.repository.Constants.STAKEHOLDER_TITLE_ENTITY_TYPE; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.getUUID; -import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.indexSearchPaginated; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.verifyDuplicateAssetByName; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; @@ -57,20 +53,15 @@ public class StakeholderTitlePreProcessor implements PreProcessor { public static final String REL_ATTR_STAKEHOLDERS = "stakeholders"; - private final AtlasGraph graph; private final AtlasTypeRegistry typeRegistry; private final EntityGraphRetriever entityRetriever; - private AtlasEntityStore entityStore; protected EntityDiscoveryService discovery; public StakeholderTitlePreProcessor(AtlasGraph graph, AtlasTypeRegistry typeRegistry, - EntityGraphRetriever entityRetriever, - AtlasEntityStore entityStore) { - this.graph = graph; + EntityGraphRetriever entityRetriever) { this.typeRegistry = typeRegistry; this.entityRetriever = entityRetriever; - this.entityStore = entityStore; try { this.discovery = new EntityDiscoveryService(typeRegistry, graph, null, null, null, null); @@ -83,7 +74,7 @@ public StakeholderTitlePreProcessor(AtlasGraph graph, public void processAttributes(AtlasStruct entityStruct, EntityMutationContext context, EntityMutations.EntityOperation operation) throws AtlasBaseException { if (LOG.isDebugEnabled()) { - LOG.debug("StakeholderTitle.processAttributes: pre processing {}, {}", entityStruct.getAttribute(QUALIFIED_NAME), operation); + LOG.debug("StakeholderTitlePreProcessor.processAttributes: pre processing {}, {}", entityStruct.getAttribute(QUALIFIED_NAME), operation); } AtlasEntity entity = (AtlasEntity) entityStruct; @@ -102,45 +93,45 @@ private void processCreateStakeholderTitle(AtlasEntity entity) throws AtlasBaseE AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreateStakeholderTitle"); try { - if (entity.hasRelationshipAttribute(REL_ATTR_STAKEHOLDERS)) { - throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Managing Stakeholders while creating StakeholderTitle"); - } + validateRelations(entity); if (RequestContext.get().isSkipAuthorizationCheck()) { + // To create bootstrap titles with provided aualifiedName return; } String name = (String) entity.getAttribute(NAME); verifyDuplicateAssetByName(STAKEHOLDER_TITLE_ENTITY_TYPE, name, discovery, - String.format("Stakeholder title with name %s already exists", name)); + format("Stakeholder title with name %s already exists", name)); List domainQualifiedNames = null; - if (entity.hasAttribute(ATTR_DOMAIN_QUALIFIED_NAMES)) { - domainQualifiedNames = (List) entity.getAttribute(ATTR_DOMAIN_QUALIFIED_NAMES); - if (domainQualifiedNames.size() == 0) { - throw new AtlasBaseException(BAD_REQUEST, "Please pass attribute domainQualifiedNames"); + Object qNamesAsObject = entity.getAttribute(ATTR_DOMAIN_QUALIFIED_NAMES); + if (qNamesAsObject != null) { + domainQualifiedNames = (List) qNamesAsObject; } + } - } else { - throw new AtlasBaseException(BAD_REQUEST, "Please pass attribute domainQualifiedNames"); + if (CollectionUtils.isEmpty(domainQualifiedNames)) { + throw new AtlasBaseException(BAD_REQUEST, "Please provide attribute domainQualifiedNames"); } if (domainQualifiedNames.contains(STAR)) { if (domainQualifiedNames.size() > 1) { + domainQualifiedNames.clear(); domainQualifiedNames.add(STAR); entity.setAttribute(ATTR_DOMAIN_QUALIFIED_NAMES, domainQualifiedNames); } - String qualifiedName = String.format(PATTERN_QUALIFIED_NAME_ALL_DOMAINS, getUUID()); + String qualifiedName = format(PATTERN_QUALIFIED_NAME_ALL_DOMAINS, getUUID()); entity.setAttribute(QUALIFIED_NAME, qualifiedName); } else { - entity.setAttribute(QUALIFIED_NAME, String.format(PATTERN_QUALIFIED_NAME_DOMAIN, getUUID())); + entity.setAttribute(QUALIFIED_NAME, format(PATTERN_QUALIFIED_NAME_DOMAIN, getUUID())); } - authorizeDomainAccess(domainQualifiedNames, AtlasPrivilege.ENTITY_UPDATE); + authorizeDomainAccess(domainQualifiedNames); } finally { RequestContext.get().endMetricRecord(metricRecorder); @@ -152,12 +143,11 @@ private void processUpdateStakeholderTitle(EntityMutationContext context, AtlasE try { if (RequestContext.get().isSkipAuthorizationCheck()) { + // To create bootstrap titles with provided aualifiedName return; } - if (entity.hasRelationshipAttribute(REL_ATTR_STAKEHOLDERS)) { - throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Managing Stakeholders while updating StakeholderTitle"); - } + validateRelations(entity); AtlasVertex vertex = context.getVertex(entity.getGuid()); @@ -165,17 +155,22 @@ private void processUpdateStakeholderTitle(EntityMutationContext context, AtlasE String newName = (String) entity.getAttribute(NAME); if (!currentName.equals(newName)) { verifyDuplicateAssetByName(STAKEHOLDER_TITLE_ENTITY_TYPE, newName, discovery, - String.format("Stakeholder title with name %s already exists", newName)); + format("StakeholderTitle with name %s already exists", newName)); } - List domainQualifiedNames; + List domainQualifiedNames = null; if (entity.hasAttribute(ATTR_DOMAIN_QUALIFIED_NAMES)) { - domainQualifiedNames = (List) entity.getAttribute(ATTR_DOMAIN_QUALIFIED_NAMES); - } else { + Object qNamesAsObject = entity.getAttribute(ATTR_DOMAIN_QUALIFIED_NAMES); + if (qNamesAsObject != null) { + domainQualifiedNames = (List) qNamesAsObject; + } + } + + if (CollectionUtils.isEmpty(domainQualifiedNames)) { domainQualifiedNames = vertex.getListProperty(ATTR_DOMAIN_QUALIFIED_NAMES, String.class); } - authorizeDomainAccess(domainQualifiedNames, AtlasPrivilege.ENTITY_UPDATE); + authorizeDomainAccess(domainQualifiedNames); String vertexQName = vertex.getProperty(QUALIFIED_NAME, String.class); entity.setAttribute(QUALIFIED_NAME, vertexQName); @@ -191,25 +186,32 @@ public void processDelete(AtlasVertex vertex) throws AtlasBaseException { try { AtlasEntity titleEntity = entityRetriever.toAtlasEntity(vertex); - List stakeholders = (List) titleEntity.getRelationshipAttribute(REL_ATTR_STAKEHOLDERS); - Optional activeStakeholder = stakeholders.stream().filter(x -> x.getRelationshipStatus() == AtlasRelationship.Status.ACTIVE).findFirst(); - if (activeStakeholder.isPresent()) { - throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Can not delete StakeholderTitle as it has reference to Active Stakeholder"); + List stakeholders = null; + Object stakeholdersAsObject = titleEntity.getRelationshipAttribute(REL_ATTR_STAKEHOLDERS); + if (stakeholdersAsObject != null) { + stakeholders = (List) stakeholdersAsObject; } - List domainQualifiedNames = vertex.getMultiValuedProperty(ATTR_DOMAIN_QUALIFIED_NAMES, String.class); + if (CollectionUtils.isNotEmpty(stakeholders)) { + Optional activeStakeholder = stakeholders.stream().filter(x -> x.getRelationshipStatus() == AtlasRelationship.Status.ACTIVE).findFirst(); + if (activeStakeholder.isPresent()) { + throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Can not delete StakeholderTitle as it has reference to Active Stakeholder"); + } - authorizeDomainAccess(domainQualifiedNames, AtlasPrivilege.ENTITY_UPDATE); + List domainQualifiedNames = vertex.getMultiValuedProperty(ATTR_DOMAIN_QUALIFIED_NAMES, String.class); + authorizeDomainAccess(domainQualifiedNames); + } } finally { RequestContext.get().endMetricRecord(metricRecorder); } } - private void authorizeDomainAccess(List domainQualifiedNames, AtlasPrivilege permission) throws AtlasBaseException { + private void authorizeDomainAccess(List domainQualifiedNames) throws AtlasBaseException { for (String domainQualifiedName: domainQualifiedNames) { String domainQualifiedNameToAuth; + if (domainQualifiedNames.contains(STAR)) { domainQualifiedNameToAuth = "*/super"; } else { @@ -219,7 +221,13 @@ private void authorizeDomainAccess(List domainQualifiedNames, AtlasPrivi AtlasEntityHeader domainHeaderToAuth = new AtlasEntityHeader(DATA_DOMAIN_ENTITY_TYPE, mapOf(QUALIFIED_NAME, domainQualifiedNameToAuth)); AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, new AtlasEntityHeader(domainHeaderToAuth)), - "create StakeholderTitle for domain ", domainQualifiedName); + "mutate StakeholderTitle for domain ", domainQualifiedName); + } + } + + private void validateRelations(AtlasEntity entity) throws AtlasBaseException { + if (entity.hasRelationshipAttribute(REL_ATTR_STAKEHOLDERS)) { + throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Managing Stakeholders while creating/updating StakeholderTitle"); } } } From bea71b8714f45fac42c4f1a1b197be0aef3bcfc7 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Mon, 13 May 2024 16:15:54 +0530 Subject: [PATCH 178/316] DG-1235 Change attribute as per models review comment --- .../v2/preprocessor/accesscontrol/StakeholderPreProcessor.java | 2 +- .../v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java index a8c01effbfe..f9e35c6ef68 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java @@ -72,7 +72,7 @@ public class StakeholderPreProcessor extends PersonaPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(StakeholderPreProcessor.class); - public static final String ATTR_DOMAIN_QUALIFIED_NAME = "domainQualifiedName"; + public static final String ATTR_DOMAIN_QUALIFIED_NAME = "stakeholderDomainQualifiedName"; public static final String ATTR_STAKEHOLDER_TITLE_GUID = "stakeholderTitleGuid"; public static final String REL_ATTR_STAKEHOLDER_TITLE = "stakeholderTitle"; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java index 7396cdde9eb..840e4ebac65 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java @@ -16,7 +16,6 @@ import org.apache.atlas.model.instance.EntityMutations; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; -import org.apache.atlas.repository.store.graph.AtlasEntityStore; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; From 3038e046e2fe5207519155451989efd40b095e0d Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Mon, 13 May 2024 17:07:38 +0530 Subject: [PATCH 179/316] DG-1235 Fix get vertex property issue --- .../v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java index 840e4ebac65..366977621d1 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java @@ -166,7 +166,7 @@ private void processUpdateStakeholderTitle(EntityMutationContext context, AtlasE } if (CollectionUtils.isEmpty(domainQualifiedNames)) { - domainQualifiedNames = vertex.getListProperty(ATTR_DOMAIN_QUALIFIED_NAMES, String.class); + domainQualifiedNames = vertex.getMultiValuedProperty(ATTR_DOMAIN_QUALIFIED_NAMES, String.class); } authorizeDomainAccess(domainQualifiedNames); From 0c5a4113b09e0b42ac18980eadd88eb11065f9e9 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Mon, 13 May 2024 17:45:00 +0530 Subject: [PATCH 180/316] DG-1235 Final default bootstrapped titles --- .../policies/global_stakeholder-titles.json | 26 ++++++++++++++++--- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/addons/policies/global_stakeholder-titles.json b/addons/policies/global_stakeholder-titles.json index a0c3cf6b0a8..897027e69f2 100644 --- a/addons/policies/global_stakeholder-titles.json +++ b/addons/policies/global_stakeholder-titles.json @@ -4,8 +4,8 @@ "typeName": "StakeholderTitle", "attributes": { - "qualifiedName": "stakeholderTitle/default/DOMAIN_OWNERS", - "name": "Domain Owners", + "qualifiedName": "stakeholderTitle/default/DOMAIN_OWNER", + "name": "Domain Owner", "domainQualifiedNames": ["*"] } }, @@ -13,8 +13,26 @@ "typeName": "StakeholderTitle", "attributes": { - "qualifiedName": "stakeholderTitle/default/DATA_STEWARDS", - "name": "Data Stewards", + "qualifiedName": "stakeholderTitle/default/DATA_PRODUCT_OWNER", + "name": "Data Product Owner", + "domainQualifiedNames": ["*"] + } + }, + { + "typeName": "StakeholderTitle", + "attributes": + { + "qualifiedName": "stakeholderTitle/default/DATA_ENGINEER", + "name": "Data Engineer", + "domainQualifiedNames": ["*"] + } + }, + { + "typeName": "StakeholderTitle", + "attributes": + { + "qualifiedName": "stakeholderTitle/default/ARCHITECT", + "name": "Architect", "domainQualifiedNames": ["*"] } } From 93589fedd6374eb37741be10c38a8e86c89d0411 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Mon, 13 May 2024 19:25:36 +0530 Subject: [PATCH 181/316] nit: update comment --- .../atlas/repository/store/graph/v2/EntityGraphMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 52d5668b447..b6bdb7e12a4 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -3011,7 +3011,7 @@ public void repairClassificationMappings(AtlasEntityHeader entityHeader, AtlasVe } } } - //Delete all the properties + //Delete array/set properties first entityVertex.removeProperty(TRAIT_NAMES_PROPERTY_KEY); entityVertex.removeProperty(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY); From 70098f9955ad059ead208f6b83a3e7f91859fe17 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Mon, 13 May 2024 20:16:21 +0530 Subject: [PATCH 182/316] PLT-1572: fix the prop through lineage --- .../atlas/repository/store/graph/v2/EntityGraphMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index a5bf1de5d06..fb5a90d767e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -3027,7 +3027,7 @@ public void addClassifications(final EntityMutationContext context, String guid, } if (restrictPropagationThroughHierarchy == null) { - classification.setRestrictPropagationThroughLineage(RESTRICT_PROPAGATION_THROUGH_HIERARCHY_DEFAULT); + classification.setRestrictPropagationThroughHierarchy(RESTRICT_PROPAGATION_THROUGH_HIERARCHY_DEFAULT); } // set associated entity id to classification From fd93ec0c78906870d2e65c2738d4b55fb2cbd69c Mon Sep 17 00:00:00 2001 From: akshaysw Date: Mon, 13 May 2024 23:44:48 +0530 Subject: [PATCH 183/316] restructure code --- .../v2/preprocessor/PreProcessorUtils.java | 6 +-- .../datamesh/DataProductPreProcessor.java | 39 ++++++++++--------- 2 files changed, 24 insertions(+), 21 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index eeb3196b4c3..df55fded0a6 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -53,9 +53,9 @@ public class PreProcessorUtils { public static final String PARENT_DOMAIN_QN_ATTR = "parentDomainQualifiedName"; public static final String SUPER_DOMAIN_QN_ATTR = "superDomainQualifiedName"; - public static final String DAAP_VISIBILITY = "daapVisibility"; - public static final String DAAP_VISIBILITY_USERS = "daapVisibilityUsers"; - public static final String DAAP_VISIBILITY_GROUPS = "daapVisibilityGroups"; + public static final String DAAP_VISIBILITY_ATTR = "daapVisibility"; + public static final String DAAP_VISIBILITY_USERS_ATTR = "daapVisibilityUsers"; + public static final String DAAP_VISIBILITY_GROUPS_ATTR = "daapVisibilityGroups"; //Migration Constants public static final String MIGRATION = "MIGRATION:"; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index b9278ef5a6d..7b0ec088f26 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -14,7 +14,6 @@ import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; -import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -30,6 +29,7 @@ public class DataProductPreProcessor extends AbstractDomainPreProcessor { private static final String PRIVATE = "Private"; private static final String PROTECTED = "Protected"; private static final String PUBLIC = "Public"; + private static final String DATA_PRODUCT = "dataProduct"; @@ -59,7 +59,7 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co switch (operation) { case CREATE: - processCreateProduct(entity,vertex); + processCreateProduct(entity, vertex); break; case UPDATE: processUpdateProduct(entity, vertex); @@ -82,7 +82,7 @@ private void processCreateProduct(AtlasEntity entity,AtlasVertex vertex) throws productExists(productName, parentDomainQualifiedName); - createDaapVisibilityPolicy(entity,vertex); + createDaapVisibilityPolicy(entity, vertex); RequestContext.get().endMetricRecord(metricRecorder); } @@ -114,13 +114,15 @@ private void processUpdateProduct(AtlasEntity entity, AtlasVertex vertex) throws } // check for daapVisibility change - String currentProductDaapVisibility = storedProduct.getAttribute(DAAP_VISIBILITY).toString(); - String newProductDaapVisibility = (String) entity.getAttribute(DAAP_VISIBILITY); + String currentProductDaapVisibility = storedProduct.getAttribute(DAAP_VISIBILITY_ATTR).toString(); + String newProductDaapVisibility = (String) entity.getAttribute(DAAP_VISIBILITY_ATTR);// check case if attribute is not sent from FE + + boolean isDaapVisibilityChanged = (newProductDaapVisibility != null && !newProductDaapVisibility.equals(currentProductDaapVisibility)); if (newParentDomainQualifiedName != null && !newParentDomainQualifiedName.equals(currentParentDomainQualifiedName)) { - if (newProductDaapVisibility != null && !newProductDaapVisibility.equals(currentProductDaapVisibility)){ - throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Moving the product to another domain, along with the change in Dapp visibility, is not allowed"); + if(isDaapVisibilityChanged){ + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Moving the product to another domain along with the change in Daap visibility is not allowed"); } //Auth check @@ -145,8 +147,8 @@ private void processUpdateProduct(AtlasEntity entity, AtlasVertex vertex) throws entity.setAttribute(QUALIFIED_NAME, vertexQnName); } - if (newProductDaapVisibility != null && !newProductDaapVisibility.equals(currentProductDaapVisibility)) { - updateDaapVisibilityPolicy(entity, storedProduct, currentProductDaapVisibility,newProductDaapVisibility); + if (isDaapVisibilityChanged) { + updateDaapVisibilityPolicy(entity, storedProduct, currentProductDaapVisibility, newProductDaapVisibility); } RequestContext.get().endMetricRecord(metricRecorder); @@ -230,16 +232,17 @@ private void createDaapVisibilityPolicy(AtlasEntity entity,AtlasVertex vertex) t policy.setAttribute(ATTR_POLICY_RESOURCES, Arrays.asList("entity:" + entity.getAttribute(QUALIFIED_NAME))); policy.setAttribute(ATTR_POLICY_RESOURCES_CATEGORY, POLICY_RESOURCE_CATEGORY_PERSONA_ENTITY); policy.setAttribute(ATTR_POLICY_SERVICE_NAME, "atlas"); - policy.setAttribute(ATTR_POLICY_SUB_CATEGORY, DATA_PRODUCT_ENTITY_TYPE); + policy.setAttribute(ATTR_POLICY_SUB_CATEGORY, DATA_PRODUCT); // create new constant attr - switch ((String) entity.getAttribute(DAAP_VISIBILITY)) { + switch ((String) entity.getAttribute(DAAP_VISIBILITY_ATTR)) { case PRIVATE: - // do nothing for private daapVisibility + policy.setAttribute(ATTR_POLICY_USERS,Arrays.asList()); + policy.setAttribute(ATTR_POLICY_GROUPS,Arrays.asList()); break; case PROTECTED: // create policy for policyUsers and policyGroups - policy.setAttribute(ATTR_POLICY_USERS, entity.getAttribute(DAAP_VISIBILITY_USERS)); - policy.setAttribute(ATTR_POLICY_GROUPS, entity.getAttribute(DAAP_VISIBILITY_GROUPS)); + policy.setAttribute(ATTR_POLICY_USERS, entity.getAttribute(DAAP_VISIBILITY_USERS_ATTR)); + policy.setAttribute(ATTR_POLICY_GROUPS, entity.getAttribute(DAAP_VISIBILITY_GROUPS_ATTR)); break; case PUBLIC: // set empty user list @@ -272,8 +275,8 @@ private void updateDaapVisibilityPolicy(AtlasEntity newEntity, AtlasEntity curre switch (newProductDaapVisibility) { case PROTECTED: // create policy for policyUsers and policyGroups - policy.setAttribute(ATTR_POLICY_USERS, newEntity.getAttribute(DAAP_VISIBILITY_USERS)); - policy.setAttribute(ATTR_POLICY_GROUPS, newEntity.getAttribute(DAAP_VISIBILITY_GROUPS)); + policy.setAttribute(ATTR_POLICY_USERS, newEntity.getAttribute(DAAP_VISIBILITY_USERS_ATTR)); + policy.setAttribute(ATTR_POLICY_GROUPS, newEntity.getAttribute(DAAP_VISIBILITY_GROUPS_ATTR)); break; case PUBLIC: // set empty user list @@ -305,8 +308,8 @@ private void updateDaapVisibilityPolicy(AtlasEntity newEntity, AtlasEntity curre break; case PROTECTED: // create policy for policyUsers and policyGroups - policy.setAttribute(ATTR_POLICY_USERS, newEntity.getAttribute(DAAP_VISIBILITY_USERS)); - policy.setAttribute(ATTR_POLICY_GROUPS, newEntity.getAttribute(DAAP_VISIBILITY_GROUPS)); + policy.setAttribute(ATTR_POLICY_USERS, newEntity.getAttribute(DAAP_VISIBILITY_USERS_ATTR)); + policy.setAttribute(ATTR_POLICY_GROUPS, newEntity.getAttribute(DAAP_VISIBILITY_GROUPS_ATTR)); break; } break; From 965e757b8394a5757bb543e2a6597db733731323 Mon Sep 17 00:00:00 2001 From: akshaysw Date: Mon, 13 May 2024 23:57:37 +0530 Subject: [PATCH 184/316] restructure code --- .../datamesh/DataProductPreProcessor.java | 66 ++++++++----------- 1 file changed, 28 insertions(+), 38 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 7b0ec088f26..9f33aab8756 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -236,31 +236,19 @@ private void createDaapVisibilityPolicy(AtlasEntity entity,AtlasVertex vertex) t switch ((String) entity.getAttribute(DAAP_VISIBILITY_ATTR)) { case PRIVATE: - policy.setAttribute(ATTR_POLICY_USERS,Arrays.asList()); - policy.setAttribute(ATTR_POLICY_GROUPS,Arrays.asList()); + setPolicyAttributes(policy, Arrays.asList(), Arrays.asList()); break; case PROTECTED: - // create policy for policyUsers and policyGroups - policy.setAttribute(ATTR_POLICY_USERS, entity.getAttribute(DAAP_VISIBILITY_USERS_ATTR)); - policy.setAttribute(ATTR_POLICY_GROUPS, entity.getAttribute(DAAP_VISIBILITY_GROUPS_ATTR)); + setPolicyAttributes(policy, + (List) entity.getAttribute(DAAP_VISIBILITY_USERS_ATTR), + (List) entity.getAttribute(DAAP_VISIBILITY_GROUPS_ATTR) + ); break; case PUBLIC: - // set empty user list - policy.setAttribute(ATTR_POLICY_USERS, Arrays.asList()); - // set user groups oto public to allow access for all - policy.setAttribute(ATTR_POLICY_GROUPS, Arrays.asList("public")); + setPolicyAttributes(policy, Arrays.asList(), Arrays.asList("public")); break; } - - try { - RequestContext.get().setSkipAuthorizationCheck(true); - AtlasEntity.AtlasEntitiesWithExtInfo policiesExtInfo = new AtlasEntity.AtlasEntitiesWithExtInfo(); - policiesExtInfo.addEntity(policy); - EntityStream entityStream = new AtlasEntityStream(policiesExtInfo); - entityStore.createOrUpdate(entityStream, false); // adding new policy - } finally { - RequestContext.get().setSkipAuthorizationCheck(false); - } + createOrUpdatePolicy(policy); } private void updateDaapVisibilityPolicy(AtlasEntity newEntity, AtlasEntity currentEntity, String currentProductDaapVisibility, String newProductDaapVisibility) throws AtlasBaseException{ @@ -275,46 +263,42 @@ private void updateDaapVisibilityPolicy(AtlasEntity newEntity, AtlasEntity curre switch (newProductDaapVisibility) { case PROTECTED: // create policy for policyUsers and policyGroups - policy.setAttribute(ATTR_POLICY_USERS, newEntity.getAttribute(DAAP_VISIBILITY_USERS_ATTR)); - policy.setAttribute(ATTR_POLICY_GROUPS, newEntity.getAttribute(DAAP_VISIBILITY_GROUPS_ATTR)); + setPolicyAttributes(policy, + (List) newEntity.getAttribute(DAAP_VISIBILITY_USERS_ATTR), + (List) newEntity.getAttribute(DAAP_VISIBILITY_GROUPS_ATTR) + ); break; case PUBLIC: - // set empty user list - policy.setAttribute(ATTR_POLICY_USERS, Arrays.asList()); - // set user groups to public to allow access for all - policy.setAttribute(ATTR_POLICY_GROUPS, Arrays.asList("public")); + setPolicyAttributes(policy, Arrays.asList(), Arrays.asList("public")); } break; case PROTECTED: switch (newProductDaapVisibility) { case PRIVATE: - // create policy for policyUsers and policyGroups - policy.setAttribute(ATTR_POLICY_USERS,Arrays.asList()); - policy.setAttribute(ATTR_POLICY_GROUPS,Arrays.asList()); + setPolicyAttributes(policy, Arrays.asList(), Arrays.asList()); break; case PUBLIC: - // set empty user list - policy.setAttribute(ATTR_POLICY_USERS, Arrays.asList()); - // set user groups to public to allow access for all - policy.setAttribute(ATTR_POLICY_GROUPS, Arrays.asList("public")); + setPolicyAttributes(policy, Arrays.asList(), Arrays.asList("public")); } break; case PUBLIC: switch (newProductDaapVisibility) { case PRIVATE: - // create policy for policyUsers and policyGroups - policy.setAttribute(ATTR_POLICY_USERS,Arrays.asList()); - policy.setAttribute(ATTR_POLICY_GROUPS,Arrays.asList()); + setPolicyAttributes(policy, Arrays.asList(), Arrays.asList()); break; case PROTECTED: - // create policy for policyUsers and policyGroups - policy.setAttribute(ATTR_POLICY_USERS, newEntity.getAttribute(DAAP_VISIBILITY_USERS_ATTR)); - policy.setAttribute(ATTR_POLICY_GROUPS, newEntity.getAttribute(DAAP_VISIBILITY_GROUPS_ATTR)); + setPolicyAttributes(policy, + (List) newEntity.getAttribute(DAAP_VISIBILITY_USERS_ATTR), + (List) newEntity.getAttribute(DAAP_VISIBILITY_GROUPS_ATTR) + ); break; } break; } + createOrUpdatePolicy(policy); + } + private void createOrUpdatePolicy(AtlasEntity policy) throws AtlasBaseException{ try { RequestContext.get().setSkipAuthorizationCheck(true); AtlasEntity.AtlasEntitiesWithExtInfo policiesExtInfo = new AtlasEntity.AtlasEntitiesWithExtInfo(); @@ -325,4 +309,10 @@ private void updateDaapVisibilityPolicy(AtlasEntity newEntity, AtlasEntity curre RequestContext.get().setSkipAuthorizationCheck(false); } } + + // Helper method to set policy attributes + private void setPolicyAttributes(AtlasEntity policy, List users, List groups) { + policy.setAttribute(ATTR_POLICY_USERS, users); + policy.setAttribute(ATTR_POLICY_GROUPS, groups); + } } From 97291e2ea0d7b4ee86e25cf5bc357d58b421b8bb Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Tue, 14 May 2024 10:44:37 +0530 Subject: [PATCH 185/316] DG-1235 StakeholderTitle parent to Asset --- addons/models/0000-Area0/0010-base_model.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/addons/models/0000-Area0/0010-base_model.json b/addons/models/0000-Area0/0010-base_model.json index 5f21227e3dc..1b17807b827 100644 --- a/addons/models/0000-Area0/0010-base_model.json +++ b/addons/models/0000-Area0/0010-base_model.json @@ -1046,7 +1046,7 @@ "name": "StakeholderTitle", "description": "Instance of a stakeholder title for Domains in Atlan", "superTypes": [ - "Catalog" + "Asset" ], "serviceType": "atlan", "typeVersion": "1.0", From 56a851e04a6cdd5f2510adebab1495483a886e71 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Tue, 14 May 2024 10:53:29 +0530 Subject: [PATCH 186/316] DG-1235 Remove redundant policy from transformer --- .../policy_cache_transformer_persona.json | 34 +------------------ 1 file changed, 1 insertion(+), 33 deletions(-) diff --git a/addons/static/templates/policy_cache_transformer_persona.json b/addons/static/templates/policy_cache_transformer_persona.json index fdcf4e90249..a9bde84febd 100644 --- a/addons/static/templates/policy_cache_transformer_persona.json +++ b/addons/static/templates/policy_cache_transformer_persona.json @@ -422,7 +422,7 @@ { "policyResourceCategory": "RELATIONSHIP", "policyType": "ACCESS", - "description": "Link/unlink any Stakeholder to this Domain", + "description": "Link/unlink Stakeholder to this Domain", "resources": [ "relationship-type:*", @@ -455,22 +455,6 @@ ], "actions": ["add-relationship", "update-relationship", "remove-relationship"] }, - { - "policyResourceCategory": "ENTITY", - "policyType": "ACCESS", - "description": "Create Stakeholder Title for this Domain", - - "resources": [ - "entity:stakeholderTitle/domain/*/{entity}", - "entity-type:StakeholderTitle", - "entity-classification:*" - ], - "actions": [ - "entity-create", - "entity-update", - "entity-delete" - ] - }, { "policyResourceCategory": "RELATIONSHIP", "policyType": "ACCESS", @@ -661,22 +645,6 @@ ], "actions": ["add-relationship", "update-relationship", "remove-relationship"] }, - { - "policyResourceCategory": "ENTITY", - "policyType": "ACCESS", - "description": "Create Stakeholder Title for Sub Domains", - - "resources": [ - "entity:stakeholderTitle/domain/*/{entity}/*domain/*", - "entity-type:StakeholderTitle", - "entity-classification:*" - ], - "actions": [ - "entity-create", - "entity-update", - "entity-delete" - ] - }, { "policyResourceCategory": "RELATIONSHIP", "policyType": "ACCESS", From dafe73f32cc31b2c97adb34429a934d96e9f0d9a Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Tue, 14 May 2024 12:28:45 +0530 Subject: [PATCH 187/316] DG-1235 Rename attribute domainQualifiedNames to stakeholderTitleDomainQualifiedNames --- addons/models/0000-Area0/0010-base_model.json | 2 +- addons/policies/global_stakeholder-titles.json | 8 ++++---- .../accesscontrol/StakeholderPreProcessor.java | 2 +- .../datamesh/StakeholderTitlePreProcessor.java | 2 +- .../apache/atlas/repository/util/AccessControlUtils.java | 4 ---- 5 files changed, 7 insertions(+), 11 deletions(-) diff --git a/addons/models/0000-Area0/0010-base_model.json b/addons/models/0000-Area0/0010-base_model.json index 1b17807b827..662db6179f8 100644 --- a/addons/models/0000-Area0/0010-base_model.json +++ b/addons/models/0000-Area0/0010-base_model.json @@ -1052,7 +1052,7 @@ "typeVersion": "1.0", "attributeDefs": [ { - "name": "domainQualifiedNames", + "name": "stakeholderTitleDomainQualifiedNames", "description": "qualified name array representing the Domains for which this StakeholderTitle is applicable", "typeName": "array", "indexType": "STRING", diff --git a/addons/policies/global_stakeholder-titles.json b/addons/policies/global_stakeholder-titles.json index 897027e69f2..4fc24e102a3 100644 --- a/addons/policies/global_stakeholder-titles.json +++ b/addons/policies/global_stakeholder-titles.json @@ -6,7 +6,7 @@ { "qualifiedName": "stakeholderTitle/default/DOMAIN_OWNER", "name": "Domain Owner", - "domainQualifiedNames": ["*"] + "stakeholderTitleDomainQualifiedNames": ["*"] } }, { @@ -15,7 +15,7 @@ { "qualifiedName": "stakeholderTitle/default/DATA_PRODUCT_OWNER", "name": "Data Product Owner", - "domainQualifiedNames": ["*"] + "stakeholderTitleDomainQualifiedNames": ["*"] } }, { @@ -24,7 +24,7 @@ { "qualifiedName": "stakeholderTitle/default/DATA_ENGINEER", "name": "Data Engineer", - "domainQualifiedNames": ["*"] + "stakeholderTitleDomainQualifiedNames": ["*"] } }, { @@ -33,7 +33,7 @@ { "qualifiedName": "stakeholderTitle/default/ARCHITECT", "name": "Architect", - "domainQualifiedNames": ["*"] + "stakeholderTitleDomainQualifiedNames": ["*"] } } ] diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java index f9e35c6ef68..e14e47957e8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java @@ -268,7 +268,7 @@ protected void verifyDuplicateStakeholderByDomainAndTitle(String domainQualified if (CollectionUtils.isNotEmpty(assets)) { throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, - format("Stakeholder for provided title & domain combination already exists with name: {}", assets.get(0).getAttribute(NAME))); + format("Stakeholder for provided title & domain combination already exists with name: %s", assets.get(0).getAttribute(NAME))); } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java index 366977621d1..3e98b44051a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java @@ -48,7 +48,7 @@ public class StakeholderTitlePreProcessor implements PreProcessor { public static final String STAR = "*"; - public static final String ATTR_DOMAIN_QUALIFIED_NAMES = "domainQualifiedNames"; + public static final String ATTR_DOMAIN_QUALIFIED_NAMES = "stakeholderTitleDomainQualifiedNames"; public static final String REL_ATTR_STAKEHOLDERS = "stakeholders"; diff --git a/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java b/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java index 4ebf8c8a661..c2c04b8d4ed 100644 --- a/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java @@ -18,7 +18,6 @@ package org.apache.atlas.repository.util; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.featureflag.FeatureFlagStore; import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntityHeader; @@ -28,11 +27,9 @@ import org.apache.atlas.repository.graphdb.AtlasIndexQuery; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.graphdb.DirectIndexQueryResult; -import org.apache.atlas.repository.store.graph.AtlasEntityStore; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.util.NanoIdUtils; import org.apache.commons.collections.CollectionUtils; -import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,7 +41,6 @@ import java.util.stream.Collectors; import static org.apache.atlas.AtlasErrorCode.ACCESS_CONTROL_ALREADY_EXISTS; -import static org.apache.atlas.AtlasErrorCode.DISABLED_OPERATION; import static org.apache.atlas.AtlasErrorCode.OPERATION_NOT_SUPPORTED; import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.util.AtlasEntityUtils.getListAttribute; From 1a5a7aea0b9a8c4f954a736a1e13320f1dc3e244 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Tue, 14 May 2024 12:41:33 +0530 Subject: [PATCH 188/316] DG-1235 Rename relatonship endDef name dataDomain to stakeholderDataDomain --- .../v2/preprocessor/accesscontrol/StakeholderPreProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java index e14e47957e8..a43dfc64cc5 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java @@ -76,7 +76,7 @@ public class StakeholderPreProcessor extends PersonaPreProcessor { public static final String ATTR_STAKEHOLDER_TITLE_GUID = "stakeholderTitleGuid"; public static final String REL_ATTR_STAKEHOLDER_TITLE = "stakeholderTitle"; - public static final String REL_ATTR_STAKEHOLDER_DOMAIN = "dataDomain"; + public static final String REL_ATTR_STAKEHOLDER_DOMAIN = "stakeholderDataDomain"; protected EntityDiscoveryService discovery; From a8fda2287628d11c6685bd648db4be919bd51c45 Mon Sep 17 00:00:00 2001 From: akshaysw Date: Tue, 14 May 2024 13:01:35 +0530 Subject: [PATCH 189/316] fix update policy flow --- .../datamesh/DataProductPreProcessor.java | 40 ++++++++++++------- 1 file changed, 25 insertions(+), 15 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 9f33aab8756..7c238d0a807 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -12,6 +12,8 @@ import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; import org.apache.atlas.repository.store.graph.v2.EntityStream; import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils; +import org.apache.atlas.repository.util.AtlasEntityUtils; +import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.lang.StringUtils; @@ -248,46 +250,47 @@ private void createDaapVisibilityPolicy(AtlasEntity entity,AtlasVertex vertex) t setPolicyAttributes(policy, Arrays.asList(), Arrays.asList("public")); break; } - createOrUpdatePolicy(policy); + createPolicy(policy); } private void updateDaapVisibilityPolicy(AtlasEntity newEntity, AtlasEntity currentEntity, String currentProductDaapVisibility, String newProductDaapVisibility) throws AtlasBaseException{ - - AtlasEntity policy = new AtlasEntity(); - policy.setTypeName(POLICY_ENTITY_TYPE); - policy.setAttribute(NAME,newEntity.getAttribute(NAME)); - policy.setAttribute(QUALIFIED_NAME,currentEntity.getGuid()+"/read-policy"); + AtlasObjectId atlasObjectId = new AtlasObjectId(); + atlasObjectId.setTypeName(POLICY_ENTITY_TYPE); + atlasObjectId.setUniqueAttributes(AtlasEntityUtils.mapOf(QUALIFIED_NAME,currentEntity.getGuid()+"/read-policy")); + AtlasVertex policyVertex = entityRetriever.getEntityVertex(atlasObjectId); + AtlasEntity policy = entityRetriever.toAtlasEntity(policyVertex); + Map updatedAttributes = new HashMap<>(); switch (currentProductDaapVisibility) { case PRIVATE: switch (newProductDaapVisibility) { case PROTECTED: // create policy for policyUsers and policyGroups - setPolicyAttributes(policy, + updatedAttributes = setPolicyAttributes(policy, (List) newEntity.getAttribute(DAAP_VISIBILITY_USERS_ATTR), (List) newEntity.getAttribute(DAAP_VISIBILITY_GROUPS_ATTR) ); break; case PUBLIC: - setPolicyAttributes(policy, Arrays.asList(), Arrays.asList("public")); + updatedAttributes = setPolicyAttributes(policy, Arrays.asList(), Arrays.asList("public")); } break; case PROTECTED: switch (newProductDaapVisibility) { case PRIVATE: - setPolicyAttributes(policy, Arrays.asList(), Arrays.asList()); + updatedAttributes = setPolicyAttributes(policy, Arrays.asList(), Arrays.asList()); break; case PUBLIC: - setPolicyAttributes(policy, Arrays.asList(), Arrays.asList("public")); + updatedAttributes = setPolicyAttributes(policy, Arrays.asList(), Arrays.asList("public")); } break; case PUBLIC: switch (newProductDaapVisibility) { case PRIVATE: - setPolicyAttributes(policy, Arrays.asList(), Arrays.asList()); + updatedAttributes = setPolicyAttributes(policy, Arrays.asList(), Arrays.asList()); break; case PROTECTED: - setPolicyAttributes(policy, + updatedAttributes = setPolicyAttributes(policy, (List) newEntity.getAttribute(DAAP_VISIBILITY_USERS_ATTR), (List) newEntity.getAttribute(DAAP_VISIBILITY_GROUPS_ATTR) ); @@ -295,10 +298,13 @@ private void updateDaapVisibilityPolicy(AtlasEntity newEntity, AtlasEntity curre } break; } - createOrUpdatePolicy(policy); + + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(POLICY_ENTITY_TYPE); + context.addUpdated(policy.getGuid(), policy, entityType, policyVertex); + recordUpdatedChildEntities(policyVertex, updatedAttributes); } - private void createOrUpdatePolicy(AtlasEntity policy) throws AtlasBaseException{ + private void createPolicy(AtlasEntity policy) throws AtlasBaseException{ try { RequestContext.get().setSkipAuthorizationCheck(true); AtlasEntity.AtlasEntitiesWithExtInfo policiesExtInfo = new AtlasEntity.AtlasEntitiesWithExtInfo(); @@ -311,8 +317,12 @@ private void createOrUpdatePolicy(AtlasEntity policy) throws AtlasBaseException{ } // Helper method to set policy attributes - private void setPolicyAttributes(AtlasEntity policy, List users, List groups) { + private Map setPolicyAttributes(AtlasEntity policy, List users, List groups) { + Map updatedAttributes = new HashMap<>(); policy.setAttribute(ATTR_POLICY_USERS, users); policy.setAttribute(ATTR_POLICY_GROUPS, groups); + updatedAttributes.put(ATTR_POLICY_USERS, users); + updatedAttributes.put(ATTR_POLICY_GROUPS, groups); + return updatedAttributes; } } From 5f6a90110cc4709bb1ebc66501ac4f213fbe639b Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Tue, 14 May 2024 13:05:40 +0530 Subject: [PATCH 190/316] DG-1235 Refactoring --- .../store/graph/v2/AtlasEntityStoreV2.java | 2 +- .../datamesh/DataDomainPreProcessor.java | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index bdb088d48b6..a43a7bbeb12 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -1857,7 +1857,7 @@ public PreProcessor getPreProcessor(String typeName) { preProcessor = new ContractPreProcessor(graph, typeRegistry, entityRetriever, storeDifferentialAudits, discovery); break; - case "StakeholderTitle": + case STAKEHOLDER_TITLE_ENTITY_TYPE: preProcessor = new StakeholderTitlePreProcessor(graph, typeRegistry, entityRetriever); break; } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java index db1a8cc1830..b809749f000 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java @@ -84,9 +84,7 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co private void processCreateDomain(AtlasEntity entity) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreateDomain"); - if(entity.hasRelationshipAttribute(STAKEHOLDER_REL_TYPE)){ - throw new AtlasBaseException(AtlasErrorCode.OPERATION_NOT_SUPPORTED, "Managing Stakeholders while creating a domain"); - } + validateStakeholderRelationship(entity); String domainName = (String) entity.getAttribute(NAME); @@ -113,9 +111,7 @@ private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Cannot update Domain's subDomains or dataProducts relations"); } - if(entity.hasRelationshipAttribute(STAKEHOLDER_REL_TYPE)){ - throw new AtlasBaseException(AtlasErrorCode.OPERATION_NOT_SUPPORTED, "Managing Stakeholders while updating a domain"); - } + validateStakeholderRelationship(entity); String vertexQnName = vertex.getProperty(QUALIFIED_NAME, String.class); @@ -356,6 +352,12 @@ private String getOwnQualifiedNameForChild(String childQualifiedName) { String[] splitted = childQualifiedName.split("/"); return String.format("/%s/%s", splitted[splitted.length -2], splitted[splitted.length -1]); } + + private void validateStakeholderRelationship(AtlasEntity entity) throws AtlasBaseException { + if(entity.hasRelationshipAttribute(STAKEHOLDER_REL_TYPE)){ + throw new AtlasBaseException(AtlasErrorCode.OPERATION_NOT_SUPPORTED, "Managing Stakeholders while creating/updating a domain"); + } + } } From 2c30af7e23abafdbbc97fd30a51c13f999690667 Mon Sep 17 00:00:00 2001 From: akshaysw Date: Tue, 14 May 2024 14:14:51 +0530 Subject: [PATCH 191/316] do not create auth policy creation for private daap visibility --- .../v2/preprocessor/datamesh/DataProductPreProcessor.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 7c238d0a807..d2c15796dca 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -239,18 +239,20 @@ private void createDaapVisibilityPolicy(AtlasEntity entity,AtlasVertex vertex) t switch ((String) entity.getAttribute(DAAP_VISIBILITY_ATTR)) { case PRIVATE: setPolicyAttributes(policy, Arrays.asList(), Arrays.asList()); + // do not create any auth policy in case of private daap visibility break; case PROTECTED: setPolicyAttributes(policy, (List) entity.getAttribute(DAAP_VISIBILITY_USERS_ATTR), (List) entity.getAttribute(DAAP_VISIBILITY_GROUPS_ATTR) ); + createPolicy(policy); break; case PUBLIC: setPolicyAttributes(policy, Arrays.asList(), Arrays.asList("public")); + createPolicy(policy); break; } - createPolicy(policy); } private void updateDaapVisibilityPolicy(AtlasEntity newEntity, AtlasEntity currentEntity, String currentProductDaapVisibility, String newProductDaapVisibility) throws AtlasBaseException{ @@ -270,15 +272,18 @@ private void updateDaapVisibilityPolicy(AtlasEntity newEntity, AtlasEntity curre (List) newEntity.getAttribute(DAAP_VISIBILITY_USERS_ATTR), (List) newEntity.getAttribute(DAAP_VISIBILITY_GROUPS_ATTR) ); + updatedAttributes.put(ATTR_POLICY_IS_ENABLED, true); break; case PUBLIC: updatedAttributes = setPolicyAttributes(policy, Arrays.asList(), Arrays.asList("public")); + updatedAttributes.put(ATTR_POLICY_IS_ENABLED, true); } break; case PROTECTED: switch (newProductDaapVisibility) { case PRIVATE: updatedAttributes = setPolicyAttributes(policy, Arrays.asList(), Arrays.asList()); + updatedAttributes.put(ATTR_POLICY_IS_ENABLED, false); break; case PUBLIC: updatedAttributes = setPolicyAttributes(policy, Arrays.asList(), Arrays.asList("public")); @@ -288,6 +293,7 @@ private void updateDaapVisibilityPolicy(AtlasEntity newEntity, AtlasEntity curre switch (newProductDaapVisibility) { case PRIVATE: updatedAttributes = setPolicyAttributes(policy, Arrays.asList(), Arrays.asList()); + updatedAttributes.put(ATTR_POLICY_IS_ENABLED, false); break; case PROTECTED: updatedAttributes = setPolicyAttributes(policy, From 361ec0bcdd4940f1c54c679564852d76ecd81743 Mon Sep 17 00:00:00 2001 From: akshaysw Date: Tue, 14 May 2024 14:32:41 +0530 Subject: [PATCH 192/316] fix bug --- .../datamesh/DataProductPreProcessor.java | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index d2c15796dca..41d170b0194 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -119,7 +119,8 @@ private void processUpdateProduct(AtlasEntity entity, AtlasVertex vertex) throws String currentProductDaapVisibility = storedProduct.getAttribute(DAAP_VISIBILITY_ATTR).toString(); String newProductDaapVisibility = (String) entity.getAttribute(DAAP_VISIBILITY_ATTR);// check case if attribute is not sent from FE - boolean isDaapVisibilityChanged = (newProductDaapVisibility != null && !newProductDaapVisibility.equals(currentProductDaapVisibility)); + boolean isDaapVisibilityChanged = (newProductDaapVisibility != null && ((!newProductDaapVisibility.equals(currentProductDaapVisibility)) || newProductDaapVisibility.equals(PROTECTED))); + if (newParentDomainQualifiedName != null && !newParentDomainQualifiedName.equals(currentParentDomainQualifiedName)) { @@ -277,6 +278,7 @@ private void updateDaapVisibilityPolicy(AtlasEntity newEntity, AtlasEntity curre case PUBLIC: updatedAttributes = setPolicyAttributes(policy, Arrays.asList(), Arrays.asList("public")); updatedAttributes.put(ATTR_POLICY_IS_ENABLED, true); + break; } break; case PROTECTED: @@ -285,8 +287,16 @@ private void updateDaapVisibilityPolicy(AtlasEntity newEntity, AtlasEntity curre updatedAttributes = setPolicyAttributes(policy, Arrays.asList(), Arrays.asList()); updatedAttributes.put(ATTR_POLICY_IS_ENABLED, false); break; + case PROTECTED: + // create policy for policyUsers and policyGroups + updatedAttributes = setPolicyAttributes(policy, + (List) newEntity.getAttribute(DAAP_VISIBILITY_USERS_ATTR), + (List) newEntity.getAttribute(DAAP_VISIBILITY_GROUPS_ATTR) + ); + break; case PUBLIC: updatedAttributes = setPolicyAttributes(policy, Arrays.asList(), Arrays.asList("public")); + break; } break; case PUBLIC: From 339c25ce4162d19ec85fa6b3bd429d8fc5ee5f60 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Tue, 14 May 2024 15:12:52 +0530 Subject: [PATCH 193/316] DG-1235 Restrict policy creation for Stakeholder --- .../preprocessor/AuthPolicyPreProcessor.java | 19 ++++++++++++------- .../StakeholderTitlePreProcessor.java | 4 ++-- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java index 1abd4b83acc..4a963ed8738 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java @@ -24,7 +24,6 @@ import org.apache.atlas.authorize.AtlasEntityAccessRequest; import org.apache.atlas.authorize.AtlasPrivilege; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.featureflag.FeatureFlagStore; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; import org.apache.atlas.model.instance.AtlasEntityHeader; @@ -52,6 +51,7 @@ import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; import static org.apache.atlas.AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND; import static org.apache.atlas.AtlasErrorCode.INSTANCE_GUID_NOT_FOUND; +import static org.apache.atlas.AtlasErrorCode.OPERATION_NOT_SUPPORTED; import static org.apache.atlas.AtlasErrorCode.RESOURCE_NOT_FOUND; import static org.apache.atlas.AtlasErrorCode.UNAUTHORIZED_CONNECTION_ADMIN; import static org.apache.atlas.authorize.AtlasAuthorizationUtils.getCurrentUserName; @@ -61,6 +61,7 @@ import static org.apache.atlas.repository.Constants.ATTR_ADMIN_ROLES; import static org.apache.atlas.repository.Constants.KEYCLOAK_ROLE_ADMIN; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; +import static org.apache.atlas.repository.Constants.STAKEHOLDER_ENTITY_TYPE; import static org.apache.atlas.repository.util.AccessControlUtils.*; import static org.apache.atlas.repository.util.AccessControlUtils.getPolicySubCategory; @@ -105,6 +106,10 @@ private void processCreatePolicy(AtlasStruct entity) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreatePolicy"); AtlasEntity policy = (AtlasEntity) entity; + AtlasEntityWithExtInfo parent = getAccessControlEntity(policy); + AtlasEntity parentEntity = parent.getEntity(); + verifyParentTypeName(parentEntity); + String policyCategory = getPolicyCategory(policy); if (StringUtils.isEmpty(policyCategory)) { throw new AtlasBaseException(BAD_REQUEST, "Please provide attribute " + ATTR_POLICY_CATEGORY); @@ -114,9 +119,6 @@ private void processCreatePolicy(AtlasStruct entity) throws AtlasBaseException { AuthPolicyValidator validator = new AuthPolicyValidator(entityRetriever); if (POLICY_CATEGORY_PERSONA.equals(policyCategory)) { - AtlasEntityWithExtInfo parent = getAccessControlEntity(policy); - AtlasEntity parentEntity = parent.getEntity(); - String policySubCategory = getPolicySubCategory(policy); if (!POLICY_SUB_CATEGORY_DOMAIN.equals(policySubCategory)) { @@ -139,9 +141,6 @@ private void processCreatePolicy(AtlasStruct entity) throws AtlasBaseException { aliasStore.updateAlias(parent, policy); } else if (POLICY_CATEGORY_PURPOSE.equals(policyCategory)) { - AtlasEntityWithExtInfo parent = getAccessControlEntity(policy); - AtlasEntity parentEntity = parent.getEntity(); - policy.setAttribute(QUALIFIED_NAME, String.format("%s/%s", getEntityQualifiedName(parentEntity), getUUID())); validator.validate(policy, null, parentEntity, CREATE); @@ -320,4 +319,10 @@ private AtlasEntityWithExtInfo getAccessControlEntity(AtlasEntity entity) throws RequestContext.get().endMetricRecord(metricRecorder); return ret; } + + private void verifyParentTypeName(AtlasEntity parentEntity) throws AtlasBaseException { + if (parentEntity.getTypeName().equals(STAKEHOLDER_ENTITY_TYPE)) { + throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Updating policies for " + STAKEHOLDER_ENTITY_TYPE); + } + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java index 3e98b44051a..8a68e428aa7 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java @@ -95,7 +95,7 @@ private void processCreateStakeholderTitle(AtlasEntity entity) throws AtlasBaseE validateRelations(entity); if (RequestContext.get().isSkipAuthorizationCheck()) { - // To create bootstrap titles with provided aualifiedName + // To create bootstrap titles with provided qualifiedName return; } @@ -112,7 +112,7 @@ private void processCreateStakeholderTitle(AtlasEntity entity) throws AtlasBaseE } if (CollectionUtils.isEmpty(domainQualifiedNames)) { - throw new AtlasBaseException(BAD_REQUEST, "Please provide attribute domainQualifiedNames"); + throw new AtlasBaseException(BAD_REQUEST, "Please provide attribute " + ATTR_DOMAIN_QUALIFIED_NAMES); } if (domainQualifiedNames.contains(STAR)) { From 52e509c4a3b37d7eb5eb220019f88df59107930f Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Tue, 14 May 2024 15:28:52 +0530 Subject: [PATCH 194/316] DG-1235 Restrict policy relations via Relationship API --- common/src/main/java/org/apache/atlas/repository/Constants.java | 1 + .../repository/store/graph/v2/AtlasRelationshipStoreV2.java | 2 ++ 2 files changed, 3 insertions(+) diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 8fb5327a7d0..95a6e0c0078 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -167,6 +167,7 @@ public final class Constants { public static final String PURPOSE_ENTITY_TYPE = "Purpose"; public static final String POLICY_ENTITY_TYPE = "AuthPolicy"; public static final String SERVICE_ENTITY_TYPE = "AuthService"; + public static final String REL_POLICY_TO_ACCESS_CONTROL = "access_control_policies"; /** * Resource diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java index e2453ba1b43..14599421920 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java @@ -76,6 +76,7 @@ import static org.apache.atlas.repository.Constants.REL_DOMAIN_TO_DOMAINS; import static org.apache.atlas.repository.Constants.REL_DOMAIN_TO_PRODUCTS; import static org.apache.atlas.repository.Constants.REL_DOMAIN_TO_STAKEHOLDERS; +import static org.apache.atlas.repository.Constants.REL_POLICY_TO_ACCESS_CONTROL; import static org.apache.atlas.repository.Constants.REL_STAKEHOLDER_TITLE_TO_STAKEHOLDERS; import static org.apache.atlas.repository.Constants.VERSION_PROPERTY_KEY; import static org.apache.atlas.repository.graph.GraphHelper.getTypeName; @@ -114,6 +115,7 @@ public class AtlasRelationshipStoreV2 implements AtlasRelationshipStore { add(REL_DOMAIN_TO_PRODUCTS); add(REL_DOMAIN_TO_STAKEHOLDERS); add(REL_STAKEHOLDER_TITLE_TO_STAKEHOLDERS); + add(REL_POLICY_TO_ACCESS_CONTROL); }}; public enum RelationshipMutation { From 2b1c77023b69aaf37ded2eeab4173f35dfdf5b8e Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Tue, 14 May 2024 21:23:16 +0530 Subject: [PATCH 195/316] fix npe in policy preprocessor --- .../graph/v2/preprocessor/AuthPolicyPreProcessor.java | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java index 4a963ed8738..62adf8119ad 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java @@ -107,8 +107,11 @@ private void processCreatePolicy(AtlasStruct entity) throws AtlasBaseException { AtlasEntity policy = (AtlasEntity) entity; AtlasEntityWithExtInfo parent = getAccessControlEntity(policy); - AtlasEntity parentEntity = parent.getEntity(); - verifyParentTypeName(parentEntity); + AtlasEntity parentEntity = null; + if (parent != null) { + parentEntity = parent.getEntity(); + verifyParentTypeName(parentEntity); + } String policyCategory = getPolicyCategory(policy); if (StringUtils.isEmpty(policyCategory)) { From b3212e6e24d4250b5e59263258af67e66130fbfc Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Wed, 15 May 2024 14:46:19 +0530 Subject: [PATCH 196/316] fix:persona read on super domains --- .../apache/atlas/repository/store/aliasstore/ESAliasStore.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/aliasstore/ESAliasStore.java b/repository/src/main/java/org/apache/atlas/repository/store/aliasstore/ESAliasStore.java index cd53a5c279f..1991106b965 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/aliasstore/ESAliasStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/aliasstore/ESAliasStore.java @@ -213,7 +213,7 @@ private void personaPolicyToESDslClauses(List policies, for (String asset : assets) { terms.add(asset); - allowClauseList.add(mapOf("wildcard", mapOf(QUALIFIED_NAME, asset + "/*"))); + allowClauseList.add(mapOf("wildcard", mapOf(QUALIFIED_NAME, asset + "*"))); } } else if (getPolicyActions(policy).contains(ACCESS_READ_PERSONA_SUB_DOMAIN)) { From a3dad8b9b773600829bcd003aca137d1521a5c3e Mon Sep 17 00:00:00 2001 From: akshaysw Date: Wed, 15 May 2024 15:42:39 +0530 Subject: [PATCH 197/316] refactor and bug fix --- .../datamesh/DataProductPreProcessor.java | 216 +++++++++++------- 1 file changed, 133 insertions(+), 83 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 41d170b0194..e42e6203885 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -1,6 +1,7 @@ package org.apache.atlas.repository.store.graph.v2.preprocessor.datamesh; import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.DeleteType; import org.apache.atlas.RequestContext; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.*; @@ -16,6 +17,7 @@ import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; +import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -115,12 +117,7 @@ private void processUpdateProduct(AtlasEntity entity, AtlasVertex vertex) throws newParentDomainQualifiedName = (String) newParentDomainHeader.getAttribute(QUALIFIED_NAME); } - // check for daapVisibility change - String currentProductDaapVisibility = storedProduct.getAttribute(DAAP_VISIBILITY_ATTR).toString(); - String newProductDaapVisibility = (String) entity.getAttribute(DAAP_VISIBILITY_ATTR);// check case if attribute is not sent from FE - - boolean isDaapVisibilityChanged = (newProductDaapVisibility != null && ((!newProductDaapVisibility.equals(currentProductDaapVisibility)) || newProductDaapVisibility.equals(PROTECTED))); - + boolean isDaapVisibilityChanged = isDaapVisibilityChanged(storedProduct, entity); if (newParentDomainQualifiedName != null && !newParentDomainQualifiedName.equals(currentParentDomainQualifiedName)) { @@ -151,9 +148,8 @@ private void processUpdateProduct(AtlasEntity entity, AtlasVertex vertex) throws } if (isDaapVisibilityChanged) { - updateDaapVisibilityPolicy(entity, storedProduct, currentProductDaapVisibility, newProductDaapVisibility); + updateDaapVisibilityPolicy(entity, storedProduct); } - RequestContext.get().endMetricRecord(metricRecorder); } @@ -222,9 +218,7 @@ private static String createQualifiedName(String parentDomainQualifiedName) thro } - private void createDaapVisibilityPolicy(AtlasEntity entity,AtlasVertex vertex) throws AtlasBaseException { - String productGuid = vertex.getProperty("__guid", String.class); - + private AtlasEntity getPolicyEntity(AtlasEntity entity, String productGuid ) { AtlasEntity policy = new AtlasEntity(); policy.setTypeName(POLICY_ENTITY_TYPE); policy.setAttribute(NAME, entity.getAttribute(NAME)); @@ -237,87 +231,60 @@ private void createDaapVisibilityPolicy(AtlasEntity entity,AtlasVertex vertex) t policy.setAttribute(ATTR_POLICY_SERVICE_NAME, "atlas"); policy.setAttribute(ATTR_POLICY_SUB_CATEGORY, DATA_PRODUCT); // create new constant attr - switch ((String) entity.getAttribute(DAAP_VISIBILITY_ATTR)) { - case PRIVATE: - setPolicyAttributes(policy, Arrays.asList(), Arrays.asList()); - // do not create any auth policy in case of private daap visibility - break; - case PROTECTED: - setPolicyAttributes(policy, - (List) entity.getAttribute(DAAP_VISIBILITY_USERS_ATTR), - (List) entity.getAttribute(DAAP_VISIBILITY_GROUPS_ATTR) - ); - createPolicy(policy); - break; - case PUBLIC: - setPolicyAttributes(policy, Arrays.asList(), Arrays.asList("public")); - createPolicy(policy); - break; + return policy; + } + + private void createDaapVisibilityPolicy(AtlasEntity entity,AtlasVertex vertex) throws AtlasBaseException { + String productGuid = vertex.getProperty("__guid", String.class); + String vis = AtlasEntityUtils.getStringAttribute(entity,DAAP_VISIBILITY_ATTR); + + if (!vis.equals(PRIVATE)) { + AtlasEntity policy = getPolicyEntity(entity, productGuid); + + switch (vis) { + case PROTECTED: + setProtectedPolicyAttributes(policy, AtlasEntityUtils.getListAttribute(entity,DAAP_VISIBILITY_USERS_ATTR), + AtlasEntityUtils.getListAttribute(entity,DAAP_VISIBILITY_GROUPS_ATTR) + ); + break; + case PUBLIC: + setPublicPolicyAttributes(policy); + break; + } + createPolicy(policy); } } - private void updateDaapVisibilityPolicy(AtlasEntity newEntity, AtlasEntity currentEntity, String currentProductDaapVisibility, String newProductDaapVisibility) throws AtlasBaseException{ + private void updateDaapVisibilityPolicy(AtlasEntity newEntity, AtlasEntity currentEntity) throws AtlasBaseException{ + String newProductDaapVisibility = AtlasEntityUtils.getStringAttribute(newEntity,DAAP_VISIBILITY_ATTR);// check case if attribute is not sent from FE AtlasObjectId atlasObjectId = new AtlasObjectId(); atlasObjectId.setTypeName(POLICY_ENTITY_TYPE); atlasObjectId.setUniqueAttributes(AtlasEntityUtils.mapOf(QUALIFIED_NAME,currentEntity.getGuid()+"/read-policy")); AtlasVertex policyVertex = entityRetriever.getEntityVertex(atlasObjectId); - AtlasEntity policy = entityRetriever.toAtlasEntity(policyVertex); + + AtlasEntity policy; + if (policyVertex == null) { + policy = getPolicyEntity(newEntity, newEntity.getGuid()); + } else { + policy = entityRetriever.toAtlasEntity(policyVertex); + } + Map updatedAttributes = new HashMap<>(); - switch (currentProductDaapVisibility) { - case PRIVATE: - switch (newProductDaapVisibility) { - case PROTECTED: - // create policy for policyUsers and policyGroups - updatedAttributes = setPolicyAttributes(policy, - (List) newEntity.getAttribute(DAAP_VISIBILITY_USERS_ATTR), - (List) newEntity.getAttribute(DAAP_VISIBILITY_GROUPS_ATTR) - ); - updatedAttributes.put(ATTR_POLICY_IS_ENABLED, true); - break; - case PUBLIC: - updatedAttributes = setPolicyAttributes(policy, Arrays.asList(), Arrays.asList("public")); - updatedAttributes.put(ATTR_POLICY_IS_ENABLED, true); - break; - } - break; - case PROTECTED: - switch (newProductDaapVisibility) { - case PRIVATE: - updatedAttributes = setPolicyAttributes(policy, Arrays.asList(), Arrays.asList()); - updatedAttributes.put(ATTR_POLICY_IS_ENABLED, false); - break; - case PROTECTED: - // create policy for policyUsers and policyGroups - updatedAttributes = setPolicyAttributes(policy, - (List) newEntity.getAttribute(DAAP_VISIBILITY_USERS_ATTR), - (List) newEntity.getAttribute(DAAP_VISIBILITY_GROUPS_ATTR) - ); - break; - case PUBLIC: - updatedAttributes = setPolicyAttributes(policy, Arrays.asList(), Arrays.asList("public")); - break; - } - break; - case PUBLIC: - switch (newProductDaapVisibility) { - case PRIVATE: - updatedAttributes = setPolicyAttributes(policy, Arrays.asList(), Arrays.asList()); - updatedAttributes.put(ATTR_POLICY_IS_ENABLED, false); - break; - case PROTECTED: - updatedAttributes = setPolicyAttributes(policy, - (List) newEntity.getAttribute(DAAP_VISIBILITY_USERS_ATTR), - (List) newEntity.getAttribute(DAAP_VISIBILITY_GROUPS_ATTR) - ); - break; - } - break; + if (newProductDaapVisibility.equals(PRIVATE)) { + updatedAttributes = setPrivatePolicyAttributes(policy); + } + else if (newProductDaapVisibility.equals(PROTECTED)) { + updatedAttributes = setProtectedPolicyAttributes(policy, + AtlasEntityUtils.getListAttribute(newEntity,DAAP_VISIBILITY_USERS_ATTR), + AtlasEntityUtils.getListAttribute(newEntity,DAAP_VISIBILITY_GROUPS_ATTR) + ); + } + else if (newProductDaapVisibility.equals(PUBLIC)) { + updatedAttributes = setPublicPolicyAttributes(policy); } - AtlasEntityType entityType = typeRegistry.getEntityTypeByName(POLICY_ENTITY_TYPE); - context.addUpdated(policy.getGuid(), policy, entityType, policyVertex); - recordUpdatedChildEntities(policyVertex, updatedAttributes); + updatePolicy(policy, policyVertex, updatedAttributes); } private void createPolicy(AtlasEntity policy) throws AtlasBaseException{ @@ -332,13 +299,96 @@ private void createPolicy(AtlasEntity policy) throws AtlasBaseException{ } } - // Helper method to set policy attributes - private Map setPolicyAttributes(AtlasEntity policy, List users, List groups) { + private void updatePolicy(AtlasEntity policy, AtlasVertex policyVertex,Map updatedAttributes) { + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(POLICY_ENTITY_TYPE); + context.addUpdated(policy.getGuid(), policy, entityType, policyVertex); + recordUpdatedChildEntities(policyVertex, updatedAttributes); + } + + private Map setPrivatePolicyAttributes(AtlasEntity policy) { + Map updatedAttributes = new HashMap<>(); + policy.setAttribute(ATTR_POLICY_USERS, Arrays.asList()); + policy.setAttribute(ATTR_POLICY_GROUPS, Arrays.asList()); + policy.setAttribute(ATTR_POLICY_IS_ENABLED, false); + + updatedAttributes.put(ATTR_POLICY_USERS, Arrays.asList()); + updatedAttributes.put(ATTR_POLICY_GROUPS, Arrays.asList()); + updatedAttributes.put(ATTR_POLICY_IS_ENABLED, false); + + return updatedAttributes; + } + + private Map setProtectedPolicyAttributes(AtlasEntity policy, List users, List groups) { Map updatedAttributes = new HashMap<>(); policy.setAttribute(ATTR_POLICY_USERS, users); policy.setAttribute(ATTR_POLICY_GROUPS, groups); + policy.setAttribute(ATTR_POLICY_IS_ENABLED, true); + updatedAttributes.put(ATTR_POLICY_USERS, users); updatedAttributes.put(ATTR_POLICY_GROUPS, groups); + updatedAttributes.put(ATTR_POLICY_IS_ENABLED, true); + return updatedAttributes; + } + + private Map setPublicPolicyAttributes(AtlasEntity policy) { + Map updatedAttributes = new HashMap<>(); + policy.setAttribute(ATTR_POLICY_USERS, Arrays.asList()); + policy.setAttribute(ATTR_POLICY_GROUPS, Arrays.asList("public")); + policy.setAttribute(ATTR_POLICY_IS_ENABLED, true); + + updatedAttributes.put(ATTR_POLICY_USERS, Arrays.asList()); + updatedAttributes.put(ATTR_POLICY_GROUPS, Arrays.asList("public")); + updatedAttributes.put(ATTR_POLICY_IS_ENABLED, true); return updatedAttributes; } + + private Boolean isDaapVisibilityChanged(AtlasEntity storedProduct, AtlasEntity newProduct){ + + boolean isDaapVisibilityChanged = false; + // check for daapVisibility change + String currentProductDaapVisibility = AtlasEntityUtils.getStringAttribute(storedProduct, DAAP_VISIBILITY_ATTR); + String newProductDaapVisibility = AtlasEntityUtils.getStringAttribute(newProduct, DAAP_VISIBILITY_ATTR); // check case if attribute is not sent from FE + + isDaapVisibilityChanged = (newProductDaapVisibility != null && (!newProductDaapVisibility.equals(currentProductDaapVisibility))); + if(isDaapVisibilityChanged){ + return isDaapVisibilityChanged; + } + + // check if new daap visibility and old daap visibility is protected then check if any user, groups added changed + if (newProductDaapVisibility.equals(PROTECTED) && currentProductDaapVisibility.equals(PROTECTED)){ + + List storedUsers = AtlasEntityUtils.getListAttribute(storedProduct, DAAP_VISIBILITY_USERS_ATTR); + List storedGroups = AtlasEntityUtils.getListAttribute(storedProduct, DAAP_VISIBILITY_GROUPS_ATTR); + List newUsers = AtlasEntityUtils.getListAttribute(newProduct, DAAP_VISIBILITY_USERS_ATTR); + List newGroups = AtlasEntityUtils.getListAttribute(newProduct, DAAP_VISIBILITY_GROUPS_ATTR); + + isDaapVisibilityChanged = compareLists(storedUsers, newUsers) || compareLists(storedGroups, newGroups); + } + + return isDaapVisibilityChanged; + } + + public static boolean compareLists(List list1, List list2) { + return !CollectionUtils.disjunction(list1, list2).isEmpty(); + } + + @Override + public void processDelete(AtlasVertex vertex) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processProductDelete"); + + try{ + if(RequestContext.get().getDeleteType() != DeleteType.SOFT){ + String productGuid = vertex.getProperty("__guid", String.class); + AtlasObjectId atlasObjectId = new AtlasObjectId(); + atlasObjectId.setTypeName(POLICY_ENTITY_TYPE); + atlasObjectId.setUniqueAttributes(AtlasEntityUtils.mapOf(QUALIFIED_NAME, productGuid+"/read-policy")); + AtlasVertex policyVertex = entityRetriever.getEntityVertex(atlasObjectId); + entityStore.deleteById(policyVertex.getProperty("__guid", String.class)); + } + } + finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + + } } From 87f4f66257b6103e760adb74344fd74d16cb2d63 Mon Sep 17 00:00:00 2001 From: akshaysw Date: Wed, 15 May 2024 16:24:46 +0530 Subject: [PATCH 198/316] refactor and bug fix --- .../datamesh/DataProductPreProcessor.java | 32 ++++++++++++------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index e42e6203885..6fb55d8e44a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -238,14 +238,12 @@ private void createDaapVisibilityPolicy(AtlasEntity entity,AtlasVertex vertex) t String productGuid = vertex.getProperty("__guid", String.class); String vis = AtlasEntityUtils.getStringAttribute(entity,DAAP_VISIBILITY_ATTR); - if (!vis.equals(PRIVATE)) { + if (vis != null && !vis.equals(PRIVATE)){ AtlasEntity policy = getPolicyEntity(entity, productGuid); switch (vis) { case PROTECTED: - setProtectedPolicyAttributes(policy, AtlasEntityUtils.getListAttribute(entity,DAAP_VISIBILITY_USERS_ATTR), - AtlasEntityUtils.getListAttribute(entity,DAAP_VISIBILITY_GROUPS_ATTR) - ); + setProtectedPolicyAttributes(policy, entity); break; case PUBLIC: setPublicPolicyAttributes(policy); @@ -276,15 +274,18 @@ private void updateDaapVisibilityPolicy(AtlasEntity newEntity, AtlasEntity curre } else if (newProductDaapVisibility.equals(PROTECTED)) { updatedAttributes = setProtectedPolicyAttributes(policy, - AtlasEntityUtils.getListAttribute(newEntity,DAAP_VISIBILITY_USERS_ATTR), - AtlasEntityUtils.getListAttribute(newEntity,DAAP_VISIBILITY_GROUPS_ATTR) + newEntity ); } else if (newProductDaapVisibility.equals(PUBLIC)) { updatedAttributes = setPublicPolicyAttributes(policy); } - updatePolicy(policy, policyVertex, updatedAttributes); + if (policyVertex == null) { + createPolicy(policy); + } else { + updatePolicy(policy, policyVertex, updatedAttributes); + } } private void createPolicy(AtlasEntity policy) throws AtlasBaseException{ @@ -318,12 +319,15 @@ private Map setPrivatePolicyAttributes(AtlasEntity policy) { return updatedAttributes; } - private Map setProtectedPolicyAttributes(AtlasEntity policy, List users, List groups) { - Map updatedAttributes = new HashMap<>(); + private Map setProtectedPolicyAttributes(AtlasEntity policy, AtlasEntity entity) { + List users = AtlasEntityUtils.getListAttribute(entity, DAAP_VISIBILITY_USERS_ATTR); + List groups = AtlasEntityUtils.getListAttribute(entity, DAAP_VISIBILITY_GROUPS_ATTR); + policy.setAttribute(ATTR_POLICY_USERS, users); policy.setAttribute(ATTR_POLICY_GROUPS, groups); policy.setAttribute(ATTR_POLICY_IS_ENABLED, true); + Map updatedAttributes = new HashMap<>(); updatedAttributes.put(ATTR_POLICY_USERS, users); updatedAttributes.put(ATTR_POLICY_GROUPS, groups); updatedAttributes.put(ATTR_POLICY_IS_ENABLED, true); @@ -344,14 +348,18 @@ private Map setPublicPolicyAttributes(AtlasEntity policy) { private Boolean isDaapVisibilityChanged(AtlasEntity storedProduct, AtlasEntity newProduct){ - boolean isDaapVisibilityChanged = false; + boolean isDaapVisibilityChanged; // check for daapVisibility change String currentProductDaapVisibility = AtlasEntityUtils.getStringAttribute(storedProduct, DAAP_VISIBILITY_ATTR); String newProductDaapVisibility = AtlasEntityUtils.getStringAttribute(newProduct, DAAP_VISIBILITY_ATTR); // check case if attribute is not sent from FE - isDaapVisibilityChanged = (newProductDaapVisibility != null && (!newProductDaapVisibility.equals(currentProductDaapVisibility))); + if(newProductDaapVisibility == null){ + return false; + } + + isDaapVisibilityChanged = (!newProductDaapVisibility.equals(currentProductDaapVisibility)); if(isDaapVisibilityChanged){ - return isDaapVisibilityChanged; + return true; } // check if new daap visibility and old daap visibility is protected then check if any user, groups added changed From 94f3fc77436fdc6c1e6d52c09d7d8dd0f52f38fb Mon Sep 17 00:00:00 2001 From: akshaysw Date: Wed, 15 May 2024 17:19:32 +0530 Subject: [PATCH 199/316] refactor and bug fix --- .../preprocessor/datamesh/DataProductPreProcessor.java | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 6fb55d8e44a..cd13c311a3c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -258,7 +258,15 @@ private void updateDaapVisibilityPolicy(AtlasEntity newEntity, AtlasEntity curre AtlasObjectId atlasObjectId = new AtlasObjectId(); atlasObjectId.setTypeName(POLICY_ENTITY_TYPE); atlasObjectId.setUniqueAttributes(AtlasEntityUtils.mapOf(QUALIFIED_NAME,currentEntity.getGuid()+"/read-policy")); - AtlasVertex policyVertex = entityRetriever.getEntityVertex(atlasObjectId); + AtlasVertex policyVertex = null; + try { + policyVertex = entityRetriever.getEntityVertex(atlasObjectId); + } + catch(AtlasBaseException exp){ + if(!exp.getAtlasErrorCode().equals(AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND)){ + throw exp; + } + } AtlasEntity policy; if (policyVertex == null) { From bfe95434e7e177d36045d27e25cdbcaee1f01cc6 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Wed, 15 May 2024 17:54:16 +0530 Subject: [PATCH 200/316] DG-1235 Stakeholder title for all domains will have star/super instead of star --- .../v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java index 8a68e428aa7..97685d1ed62 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java @@ -47,7 +47,7 @@ public class StakeholderTitlePreProcessor implements PreProcessor { public static final String PATTERN_QUALIFIED_NAME_DOMAIN = "stakeholderTitle/domain/%s"; - public static final String STAR = "*"; + public static final String STAR = "*/super"; public static final String ATTR_DOMAIN_QUALIFIED_NAMES = "stakeholderTitleDomainQualifiedNames"; public static final String REL_ATTR_STAKEHOLDERS = "stakeholders"; From 617cc64cdbc74c44234329125d40849070d33205 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Wed, 15 May 2024 18:00:37 +0530 Subject: [PATCH 201/316] DG-1235 Stakeholder title for all domains will have star/super instead of star --- addons/policies/global_stakeholder-titles.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/addons/policies/global_stakeholder-titles.json b/addons/policies/global_stakeholder-titles.json index 4fc24e102a3..a32088f11eb 100644 --- a/addons/policies/global_stakeholder-titles.json +++ b/addons/policies/global_stakeholder-titles.json @@ -6,7 +6,7 @@ { "qualifiedName": "stakeholderTitle/default/DOMAIN_OWNER", "name": "Domain Owner", - "stakeholderTitleDomainQualifiedNames": ["*"] + "stakeholderTitleDomainQualifiedNames": ["*/super"] } }, { @@ -15,7 +15,7 @@ { "qualifiedName": "stakeholderTitle/default/DATA_PRODUCT_OWNER", "name": "Data Product Owner", - "stakeholderTitleDomainQualifiedNames": ["*"] + "stakeholderTitleDomainQualifiedNames": ["*/super"] } }, { @@ -24,7 +24,7 @@ { "qualifiedName": "stakeholderTitle/default/DATA_ENGINEER", "name": "Data Engineer", - "stakeholderTitleDomainQualifiedNames": ["*"] + "stakeholderTitleDomainQualifiedNames": ["*/super"] } }, { @@ -33,7 +33,7 @@ { "qualifiedName": "stakeholderTitle/default/ARCHITECT", "name": "Architect", - "stakeholderTitleDomainQualifiedNames": ["*"] + "stakeholderTitleDomainQualifiedNames": ["*/super"] } } ] From ae3510bc96091f0e20b6dc6c79f7df39abf34585 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Thu, 16 May 2024 01:57:29 +0530 Subject: [PATCH 202/316] DG-1235 Move domains update qualifiedName in Titles and Stakeholders --- .../StakeholderPreProcessor.java | 3 +- .../datamesh/AbstractDomainPreProcessor.java | 83 +++++++++++++++++++ .../datamesh/DataDomainPreProcessor.java | 7 +- 3 files changed, 90 insertions(+), 3 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java index a43dfc64cc5..1adbb8ec476 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java @@ -143,7 +143,8 @@ private void processCreateStakeholder(AtlasEntity entity) throws AtlasBaseExcept validateNoPoliciesAttached(entity); if (!entity.hasRelationshipAttribute(REL_ATTR_STAKEHOLDER_TITLE) || !entity.hasRelationshipAttribute(REL_ATTR_STAKEHOLDER_DOMAIN)) { - throw new AtlasBaseException(BAD_REQUEST, "Relationships stakeholderTitle and dataDomain are mandatory"); + throw new AtlasBaseException(BAD_REQUEST, + String.format("Relationships %s and %s are mandatory", REL_ATTR_STAKEHOLDER_TITLE, REL_ATTR_STAKEHOLDER_DOMAIN)); } String domainQualifiedName = getQualifiedNameFromRelationAttribute(entity, REL_ATTR_STAKEHOLDER_DOMAIN); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 936ba6d948e..1643c144a48 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -49,8 +49,11 @@ import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.*; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.accesscontrol.StakeholderPreProcessor.ATTR_DOMAIN_QUALIFIED_NAME; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.datamesh.StakeholderTitlePreProcessor.ATTR_DOMAIN_QUALIFIED_NAMES; import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_RESOURCES; import static org.apache.atlas.repository.util.AccessControlUtils.REL_ATTR_ACCESS_CONTROL; +import static org.apache.atlas.repository.util.AtlasEntityUtils.getListAttribute; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; public abstract class AbstractDomainPreProcessor implements PreProcessor { @@ -59,10 +62,12 @@ public abstract class AbstractDomainPreProcessor implements PreProcessor { protected final AtlasTypeRegistry typeRegistry; protected final EntityGraphRetriever entityRetriever; + protected EntityGraphRetriever entityRetrieverNoRelations; private final PreProcessor preProcessor; protected EntityDiscoveryService discovery; private static final Set POLICY_ATTRIBUTES_FOR_SEARCH = new HashSet<>(Arrays.asList(ATTR_POLICY_RESOURCES)); + private static final Set STAKEHOLDER_ATTRIBUTES_FOR_SEARCH = new HashSet<>(Arrays.asList(ATTR_DOMAIN_QUALIFIED_NAMES, ATTR_DOMAIN_QUALIFIED_NAME)); static final Set PARENT_ATTRIBUTES = new HashSet<>(Arrays.asList(SUPER_DOMAIN_QN_ATTR, PARENT_DOMAIN_QN_ATTR)); @@ -78,6 +83,7 @@ public abstract class AbstractDomainPreProcessor implements PreProcessor { this.preProcessor = new AuthPolicyPreProcessor(graph, typeRegistry, entityRetriever); try { + this.entityRetrieverNoRelations = new EntityGraphRetriever(graph, typeRegistry, true); this.discovery = new EntityDiscoveryService(typeRegistry, graph, null, null, null, null); } catch (AtlasException e) { e.printStackTrace(); @@ -158,6 +164,60 @@ protected void updatePolicies(Map updatedPolicyResources, Entity } } + protected void updateStakeholderTitlesAndStakeholders(Map updatedDomainQualifiedNames, EntityMutationContext context) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("updateStakeholderTitlesAndStakeholders"); + try { + + if (MapUtils.isEmpty(updatedDomainQualifiedNames)) { + return; + } + + List assets = getStakeholderTitlesAndStakeholders(updatedDomainQualifiedNames.keySet()); + + if (CollectionUtils.isNotEmpty(assets)) { + for (AtlasEntityHeader asset : assets) { + AtlasVertex vertex = entityRetrieverNoRelations.getEntityVertex(asset.getGuid()); + AtlasEntity entity = entityRetrieverNoRelations.toAtlasEntity(vertex); + Map updatedAttributes = new HashMap<>(); + AtlasEntityType entityType = null; + + if (entity.getTypeName().equals(STAKEHOLDER_ENTITY_TYPE)) { + entityType = typeRegistry.getEntityTypeByName(STAKEHOLDER_ENTITY_TYPE); + + String currentDomainQualifiedName = (String) asset.getAttribute(ATTR_DOMAIN_QUALIFIED_NAME); + + entity.setAttribute(ATTR_DOMAIN_QUALIFIED_NAME, updatedDomainQualifiedNames.get(currentDomainQualifiedName)); + updatedAttributes.put(ATTR_DOMAIN_QUALIFIED_NAME, updatedDomainQualifiedNames.get(currentDomainQualifiedName)); + + } else if (entity.getTypeName().equals(STAKEHOLDER_TITLE_ENTITY_TYPE)) { + entityType = typeRegistry.getEntityTypeByName(STAKEHOLDER_TITLE_ENTITY_TYPE); + + List currentDomainQualifiedNames = getListAttribute(asset, ATTR_DOMAIN_QUALIFIED_NAMES); + + List newDomainQualifiedNames = new ArrayList<>(); + + for (String qualifiedName : currentDomainQualifiedNames) { + if (updatedDomainQualifiedNames.containsKey(qualifiedName)) { + newDomainQualifiedNames.add(updatedDomainQualifiedNames.get(qualifiedName)); + } else { + newDomainQualifiedNames.add(qualifiedName); + } + } + + entity.setAttribute(ATTR_DOMAIN_QUALIFIED_NAMES, newDomainQualifiedNames); + updatedAttributes.put(ATTR_DOMAIN_QUALIFIED_NAME, newDomainQualifiedNames); + } + + context.addUpdated(entity.getGuid(), entity, entityType, vertex); + recordUpdatedChildEntities(vertex, updatedAttributes); + } + } + + } finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + } + protected void exists(String assetType, String assetName, String parentDomainQualifiedName) throws AtlasBaseException { boolean exists = false; @@ -217,6 +277,29 @@ protected List getPolicies(Set resources) throws Atla } } + protected List getStakeholderTitlesAndStakeholders(Set qualifiedNames) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("getStakeholderTitlesAndStakeholders"); + try { + List> mustClauseList = new ArrayList<>(); + mustClauseList.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList(STAKEHOLDER_ENTITY_TYPE, STAKEHOLDER_TITLE_ENTITY_TYPE)))); + + List> shouldClauseList = new ArrayList<>(); + shouldClauseList.add(mapOf("terms", mapOf("stakeholderTitleDomainQualifiedNames", qualifiedNames))); + shouldClauseList.add(mapOf("terms", mapOf("stakeholderDomainQualifiedName", qualifiedNames))); + + mustClauseList.add(mapOf("bool", mapOf("should", shouldClauseList))); + + Map bool = new HashMap<>(); + bool.put("must", mustClauseList); + + Map dsl = mapOf("query", mapOf("bool", bool)); + + return indexSearchPaginated(dsl, STAKEHOLDER_ATTRIBUTES_FOR_SEARCH, discovery); + } finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + } + /** * Record the updated child entities, it will be used to send notification and store audit logs * @param entityVertex Child entity vertex diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java index b809749f000..0f27ea09085 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java @@ -34,7 +34,6 @@ import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; -import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -44,18 +43,19 @@ import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.graph.GraphHelper.getActiveChildrenVertices; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.*; -import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; public class DataDomainPreProcessor extends AbstractDomainPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(DataDomainPreProcessor.class); private EntityMutationContext context; private Map updatedPolicyResources; + private Map updatedDomainQualifiedNames; public DataDomainPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, AtlasGraph graph) { super(typeRegistry, entityRetriever, graph); this.updatedPolicyResources = new HashMap<>(); + this.updatedDomainQualifiedNames = new HashMap<>(); } @Override @@ -197,9 +197,11 @@ private void processMoveSubDomainToAnotherDomain(AtlasEntity domain, String currentQualifiedName = domainVertex.getProperty(QUALIFIED_NAME, String.class); this.updatedPolicyResources.put("entity:" + currentQualifiedName, "entity:" + updatedQualifiedName); + this.updatedDomainQualifiedNames.put(currentQualifiedName, updatedQualifiedName); moveChildren(domainVertex, superDomainQualifiedName, updatedQualifiedName, sourceDomainQualifiedName, targetDomainQualifiedName); updatePolicies(this.updatedPolicyResources, this.context); + updateStakeholderTitlesAndStakeholders(this.updatedDomainQualifiedNames, this.context); LOG.info("Moved subDomain {} to Domain {}", domainName, targetDomainQualifiedName); @@ -254,6 +256,7 @@ private void moveChildrenToAnotherDomain(AtlasVertex childDomainVertex, String currentResource = "entity:"+ currentDomainQualifiedName; String updatedResource = "entity:"+ updatedDomainQualifiedName; this.updatedPolicyResources.put(currentResource, updatedResource); + this.updatedDomainQualifiedNames.put(currentDomainQualifiedName, updatedDomainQualifiedName); //update system properties GraphHelper.setModifiedByAsString(childDomainVertex, RequestContext.get().getUser()); From ac25d39e9e03b97fae7cf45a4ab2d58a8b12c79c Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Thu, 16 May 2024 02:49:24 +0530 Subject: [PATCH 203/316] DG-1235 READ Titles policy --- .../policies/bootstrap_entity_policies.json | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/addons/policies/bootstrap_entity_policies.json b/addons/policies/bootstrap_entity_policies.json index 32b0b381e1b..02260d558cd 100644 --- a/addons/policies/bootstrap_entity_policies.json +++ b/addons/policies/bootstrap_entity_policies.json @@ -3075,6 +3075,39 @@ "entity-update" ] } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "READ_DATA_MESH_STAKEHOLDER_TITLE", + "qualifiedName": "READ_DATA_MESH_STAKEHOLDER_TITLE", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:StakeholderTitle", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-read" + ] + } } ] } From e4dc8fae58a1503dae31518e67816a6b0e40aef6 Mon Sep 17 00:00:00 2001 From: akshaysw Date: Thu, 16 May 2024 10:20:39 +0530 Subject: [PATCH 204/316] add try cache in product delete method --- .../datamesh/DataProductPreProcessor.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index cd13c311a3c..4887fe9add1 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -398,8 +398,16 @@ public void processDelete(AtlasVertex vertex) throws AtlasBaseException { AtlasObjectId atlasObjectId = new AtlasObjectId(); atlasObjectId.setTypeName(POLICY_ENTITY_TYPE); atlasObjectId.setUniqueAttributes(AtlasEntityUtils.mapOf(QUALIFIED_NAME, productGuid+"/read-policy")); - AtlasVertex policyVertex = entityRetriever.getEntityVertex(atlasObjectId); - entityStore.deleteById(policyVertex.getProperty("__guid", String.class)); + AtlasVertex policyVertex; + try { + policyVertex = entityRetriever.getEntityVertex(atlasObjectId); + entityStore.deleteById(policyVertex.getProperty("__guid", String.class)); + } + catch(AtlasBaseException exp){ + if(!exp.getAtlasErrorCode().equals(AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND)){ + throw exp; + } + } } } finally { From 8b47d775214591f27a1dba0ba861c47e4146c6d3 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Thu, 16 May 2024 11:42:50 +0530 Subject: [PATCH 205/316] DG-1235 READ Stakeholders policy --- addons/static/templates/policy_cache_transformer_persona.json | 2 ++ 1 file changed, 2 insertions(+) diff --git a/addons/static/templates/policy_cache_transformer_persona.json b/addons/static/templates/policy_cache_transformer_persona.json index 6c6cdc909cb..b9541abf5be 100644 --- a/addons/static/templates/policy_cache_transformer_persona.json +++ b/addons/static/templates/policy_cache_transformer_persona.json @@ -414,6 +414,7 @@ "entity-classification:*" ], "actions": [ + "entity-read", "entity-create", "entity-update", "entity-delete" @@ -616,6 +617,7 @@ "entity-classification:*" ], "actions": [ + "entity-read", "entity-create", "entity-update", "entity-delete" From 3dd5ace673145de8deb6c731cfe6d8e38af3e4d6 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Thu, 16 May 2024 11:50:21 +0530 Subject: [PATCH 206/316] Add differential entity details for kafka event --- .../v2/preprocessor/contract/ContractPreProcessor.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index bc9ff76cbd8..038558dac23 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -193,6 +193,16 @@ private void updateExistingVersion(EntityMutationContext context, AtlasEntity en context.addUpdated(entity.getGuid(), entity, entityType, vertex); + AtlasEntityComparator entityComparator = new AtlasEntityComparator(typeRegistry, entityRetriever, context.getGuidAssignments(), true, true); + AtlasEntityComparator.AtlasEntityDiffResult diffResult = entityComparator.getDiffResult(entity, vertex, !storeDifferentialAudits); + RequestContext reqContext = RequestContext.get(); + if (diffResult.hasDifference()) { + if (storeDifferentialAudits) { + diffResult.getDiffEntity().setGuid(entity.getGuid()); + reqContext.cacheDifferentialEntity(diffResult.getDiffEntity()); + } + } + } public AtlasEntity getCurrentVersion(String datasetGuid) throws AtlasBaseException { From 81363ec1e97ff3e8ac95971d5ecc49cc65c67296 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Thu, 16 May 2024 12:31:39 +0530 Subject: [PATCH 207/316] DG-1235 Address review comment --- .../v2/preprocessor/datamesh/AbstractDomainPreProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 1643c144a48..025f2eb47f9 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -205,7 +205,7 @@ protected void updateStakeholderTitlesAndStakeholders(Map update } entity.setAttribute(ATTR_DOMAIN_QUALIFIED_NAMES, newDomainQualifiedNames); - updatedAttributes.put(ATTR_DOMAIN_QUALIFIED_NAME, newDomainQualifiedNames); + updatedAttributes.put(ATTR_DOMAIN_QUALIFIED_NAMES, newDomainQualifiedNames); } context.addUpdated(entity.getGuid(), entity, entityType, vertex); From 5cf827cb5945c7a0802cab9348fc514a265f9865 Mon Sep 17 00:00:00 2001 From: akshaysw Date: Thu, 16 May 2024 16:19:27 +0530 Subject: [PATCH 208/316] remove daap visibility entity attr for user and groups, if daapvisibility change is false --- .../v2/preprocessor/datamesh/DataProductPreProcessor.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 4887fe9add1..fb45270c736 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -150,6 +150,11 @@ private void processUpdateProduct(AtlasEntity entity, AtlasVertex vertex) throws if (isDaapVisibilityChanged) { updateDaapVisibilityPolicy(entity, storedProduct); } + else{ + // if isDaapVisibilityChanged is false, then do not update any daap visibility attributes in product entity as well + entity.removeAttribute(DAAP_VISIBILITY_USERS_ATTR); + entity.removeAttribute(DAAP_VISIBILITY_GROUPS_ATTR); + } RequestContext.get().endMetricRecord(metricRecorder); } From bc95d50ea49b9d41927f94600f4bfface950e178 Mon Sep 17 00:00:00 2001 From: suraj5077 Date: Fri, 17 May 2024 07:14:20 +0530 Subject: [PATCH 209/316] feat: LIN-902 Added outTime attribute in DFS --- .../java/org/apache/atlas/discovery/EntityLineageService.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index c9c11e4a862..7afd74b70d6 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -427,6 +427,8 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i } if (entityVertex != null && !visitedVertices.contains(getId(entityVertex))) { traverseEdgesOnDemand(entityVertex, isInput, depth - 1, nextLevel, visitedVertices, atlasLineageOnDemandContext, ret, baseGuid, entitiesTraversed, traversalOrder); // execute inner depth + AtlasEntityHeader baseEntity = ret.getGuidEntityMap().get(baseGuid); + baseEntity.setFinishTime(traversalOrder.get()); } } } From 84f300c4fa61d78727986b696d534750c99fea15 Mon Sep 17 00:00:00 2001 From: suraj5077 Date: Fri, 17 May 2024 07:24:29 +0530 Subject: [PATCH 210/316] fix: LIN-902 --- .../apache/atlas/model/instance/AtlasEntityHeader.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java index 0bd7304c58f..c251811df13 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasEntityHeader.java @@ -73,6 +73,7 @@ public class AtlasEntityHeader extends AtlasStruct implements Serializable { private String deleteHandler = null; private Integer depth = null; private Integer traversalOrder = null; + private Integer finishTime = null; private Map collapse = null; @@ -162,6 +163,14 @@ public AtlasEntityHeader(AtlasEntity entity) { public void setTraversalOrder(Integer traversalOrder) { this.traversalOrder = traversalOrder; } + public Integer getFinishTime() { + return finishTime; + } + + public void setFinishTime(Integer finishTime) { + this.finishTime = finishTime; + } + public AtlasEntity.Status getStatus() { return status; } From 8fb58994a5a603fc5350aafe5efd4d342897afcf Mon Sep 17 00:00:00 2001 From: suraj5077 Date: Fri, 17 May 2024 08:15:41 +0530 Subject: [PATCH 211/316] fix: lineage recursion fix --- .../java/org/apache/atlas/discovery/EntityLineageService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index 7afd74b70d6..0b079d9e5f2 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -427,8 +427,8 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i } if (entityVertex != null && !visitedVertices.contains(getId(entityVertex))) { traverseEdgesOnDemand(entityVertex, isInput, depth - 1, nextLevel, visitedVertices, atlasLineageOnDemandContext, ret, baseGuid, entitiesTraversed, traversalOrder); // execute inner depth - AtlasEntityHeader baseEntity = ret.getGuidEntityMap().get(baseGuid); - baseEntity.setFinishTime(traversalOrder.get()); + AtlasEntityHeader traversedEntity = ret.getGuidEntityMap().get(AtlasGraphUtilsV2.getIdFromVertex(entityVertex)); + traversedEntity.setFinishTime(traversalOrder.get()); } } } From 64bbed2fe5d0416f836feca1f4c9bc7dc3417539 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Tue, 14 May 2024 00:00:43 +0530 Subject: [PATCH 212/316] DG-1370 Write Mesh parent attributes in server --- .../datamesh/DataDomainPreProcessor.java | 31 ++++++++++++++++--- .../datamesh/DataProductPreProcessor.java | 31 ++++++++++++++----- 2 files changed, 51 insertions(+), 11 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java index 0f27ea09085..6becf855520 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java @@ -49,12 +49,14 @@ public class DataDomainPreProcessor extends AbstractDomainPreProcessor { private EntityMutationContext context; private Map updatedPolicyResources; + private EntityGraphRetriever retrieverNoRelation = null; private Map updatedDomainQualifiedNames; public DataDomainPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, AtlasGraph graph) { super(typeRegistry, entityRetriever, graph); this.updatedPolicyResources = new HashMap<>(); + this.retrieverNoRelation = new EntityGraphRetriever(graph, typeRegistry, true); this.updatedDomainQualifiedNames = new HashMap<>(); } @@ -88,14 +90,32 @@ private void processCreateDomain(AtlasEntity entity) throws AtlasBaseException { String domainName = (String) entity.getAttribute(NAME); - String parentDomainQualifiedName = (String) entity.getAttribute(PARENT_DOMAIN_QN_ATTR); + String parentDomainQualifiedName = ""; + AtlasObjectId parentDomainObject = (AtlasObjectId) entity.getRelationshipAttribute(PARENT_DOMAIN_REL_TYPE); + AtlasVertex parentDomain = null; - AtlasEntityHeader parentDomain = getParent(entity); - if(parentDomain != null ){ - parentDomainQualifiedName = (String) parentDomain.getAttribute(QUALIFIED_NAME); + if(parentDomainObject != null ){ + parentDomain = retrieverNoRelation.getEntityVertex(parentDomainObject); + parentDomainQualifiedName = parentDomain.getProperty(QUALIFIED_NAME, String.class); + } else { + entity.removeAttribute(PARENT_DOMAIN_QN_ATTR); + entity.removeAttribute(SUPER_DOMAIN_QN_ATTR); } entity.setAttribute(QUALIFIED_NAME, createQualifiedName(parentDomainQualifiedName)); + + if(StringUtils.isNotEmpty(parentDomainQualifiedName)) { + entity.setAttribute(PARENT_DOMAIN_QN_ATTR, parentDomainQualifiedName); + String superDomainQualifiedName = ""; + if(StringUtils.isEmpty(parentDomain.getProperty(SUPER_DOMAIN_QN_ATTR, String.class))){ + superDomainQualifiedName = parentDomainQualifiedName; + } + else { + superDomainQualifiedName = parentDomain.getProperty(SUPER_DOMAIN_QN_ATTR, String.class); + } + entity.setAttribute(SUPER_DOMAIN_QN_ATTR, superDomainQualifiedName); + } + entity.setCustomAttributes(customAttributes); domainExists(domainName, parentDomainQualifiedName); @@ -153,6 +173,9 @@ private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws String domainCurrentName = vertex.getProperty(NAME, String.class); String domainNewName = (String) entity.getAttribute(NAME); + entity.removeAttribute(PARENT_DOMAIN_QN_ATTR); + entity.removeAttribute(SUPER_DOMAIN_QN_ATTR); + if (!domainCurrentName.equals(domainNewName)) { domainExists(domainNewName, currentParentDomainQualifiedName); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index fb45270c736..6f25a487ae8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -18,6 +18,7 @@ import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections4.CollectionUtils; +import org.apache.commons.collections.MapUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -40,12 +41,14 @@ public class DataProductPreProcessor extends AbstractDomainPreProcessor { private EntityMutationContext context; private AtlasEntityStore entityStore; private Map updatedPolicyResources; + private EntityGraphRetriever retrieverNoRelation = null; public DataProductPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, AtlasGraph graph, AtlasEntityStore entityStore) { super(typeRegistry, entityRetriever, graph); this.updatedPolicyResources = new HashMap<>(); this.entityStore = entityStore; + this.retrieverNoRelation = new EntityGraphRetriever(graph, typeRegistry, true); } @Override @@ -73,16 +76,24 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co private void processCreateProduct(AtlasEntity entity,AtlasVertex vertex) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreateProduct"); + AtlasObjectId parentDomainObject = (AtlasObjectId) entity.getRelationshipAttribute(DATA_DOMAIN_REL_TYPE); String productName = (String) entity.getAttribute(NAME); - String parentDomainQualifiedName = (String) entity.getAttribute(PARENT_DOMAIN_QN_ATTR); + String parentDomainQualifiedName = ""; - AtlasEntityHeader parentDomain = getParent(entity); - if(parentDomain != null ){ - parentDomainQualifiedName = (String) parentDomain.getAttribute(QUALIFIED_NAME); - } + if (parentDomainObject == null) { + entity.removeAttribute(PARENT_DOMAIN_QN_ATTR); + entity.removeAttribute(SUPER_DOMAIN_QN_ATTR); + } else { + AtlasVertex parentDomain = retrieverNoRelation.getEntityVertex(parentDomainObject); + parentDomainQualifiedName = parentDomain.getProperty(QUALIFIED_NAME, String.class); + + entity.setAttribute(QUALIFIED_NAME, createQualifiedName(parentDomainQualifiedName)); - entity.setAttribute(QUALIFIED_NAME, createQualifiedName(parentDomainQualifiedName)); - entity.setCustomAttributes(customAttributes); + entity.setAttribute(PARENT_DOMAIN_QN_ATTR, parentDomainQualifiedName); + + String superDomainQualifiedName = parentDomain.getProperty(SUPER_DOMAIN_QN_ATTR, String.class); + entity.setAttribute(SUPER_DOMAIN_QN_ATTR, superDomainQualifiedName); + } productExists(productName, parentDomainQualifiedName); @@ -138,6 +149,12 @@ private void processUpdateProduct(AtlasEntity entity, AtlasVertex vertex) throws updatePolicies(this.updatedPolicyResources, this.context); } else { + entity.removeAttribute(PARENT_DOMAIN_QN_ATTR); + entity.removeAttribute(SUPER_DOMAIN_QN_ATTR); + currentParentDomainQualifiedName = ""; + if (entity.getRelationshipAttributes() != null) { + entity.getRelationshipAttributes().remove(DATA_PRODUCT_REL_TYPE); + } String productCurrentName = vertex.getProperty(NAME, String.class); String productNewName = (String) entity.getAttribute(NAME); From b1385117d9d10e8be5086517e961a2bf3b9c05de Mon Sep 17 00:00:00 2001 From: hr2904 Date: Tue, 14 May 2024 17:39:39 +0530 Subject: [PATCH 213/316] Added a fix for Entity updation, if adding a relationship to a only QN based hierarchy, instead of getting the error Duplicate exists (self referential issue), it allows addition of the new reltionship. --- .../datamesh/AbstractDomainPreProcessor.java | 13 ++++++++----- .../datamesh/DataDomainPreProcessor.java | 15 ++++++++------- .../datamesh/DataProductPreProcessor.java | 15 ++++++++------- 3 files changed, 24 insertions(+), 19 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index 025f2eb47f9..f39279fc537 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -218,26 +218,29 @@ protected void updateStakeholderTitlesAndStakeholders(Map update } } - protected void exists(String assetType, String assetName, String parentDomainQualifiedName) throws AtlasBaseException { + protected void exists(String assetType, String assetName, String parentDomainQualifiedName, String guid) throws AtlasBaseException { boolean exists = false; List> mustClauseList = new ArrayList(); mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", assetType))); mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); mustClauseList.add(mapOf("term", mapOf("name.keyword", assetName))); - + List> mustNotClauseList = new ArrayList(); + if(Objects.nonNull(guid) && StringUtils.isNotEmpty(guid)){ + mustNotClauseList.add(mapOf("term", mapOf("__guid", guid))); + } Map bool = new HashMap<>(); if (StringUtils.isNotEmpty(parentDomainQualifiedName)) { mustClauseList.add(mapOf("term", mapOf("parentDomainQualifiedName", parentDomainQualifiedName))); } else { - List> mustNotClauseList = new ArrayList(); mustNotClauseList.add(mapOf("exists", mapOf("field", "parentDomainQualifiedName"))); - bool.put("must_not", mustNotClauseList); } bool.put("must", mustClauseList); - + if(!mustNotClauseList.isEmpty()) { + bool.put("must_not", mustNotClauseList); + } Map dsl = mapOf("query", mapOf("bool", bool)); List assets = indexSearchPaginated(dsl, null, this.discovery); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java index 6becf855520..c755bdebf1f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java @@ -118,7 +118,7 @@ private void processCreateDomain(AtlasEntity entity) throws AtlasBaseException { entity.setCustomAttributes(customAttributes); - domainExists(domainName, parentDomainQualifiedName); + domainExists(domainName, parentDomainQualifiedName, null); RequestContext.get().endMetricRecord(metricRecorder); } @@ -167,7 +167,7 @@ private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws //Auth check isAuthorized(currentParentDomainHeader, newParentDomainHeader); - processMoveSubDomainToAnotherDomain(entity, vertex, currentParentDomainQualifiedName, newParentDomainQualifiedName, vertexQnName, newSuperDomainQualifiedName); + processMoveSubDomainToAnotherDomain(entity, vertex, currentParentDomainQualifiedName, newParentDomainQualifiedName, vertexQnName, newSuperDomainQualifiedName, storedDomain.getGuid()); } else { String domainCurrentName = vertex.getProperty(NAME, String.class); @@ -177,7 +177,7 @@ private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws entity.removeAttribute(SUPER_DOMAIN_QN_ATTR); if (!domainCurrentName.equals(domainNewName)) { - domainExists(domainNewName, currentParentDomainQualifiedName); + domainExists(domainNewName, currentParentDomainQualifiedName, storedDomain.getGuid()); } entity.setAttribute(QUALIFIED_NAME, vertexQnName); } @@ -190,7 +190,8 @@ private void processMoveSubDomainToAnotherDomain(AtlasEntity domain, String sourceDomainQualifiedName, String targetDomainQualifiedName, String currentDomainQualifiedName, - String superDomainQualifiedName) throws AtlasBaseException { + String superDomainQualifiedName, + String guid) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("processMoveSubDomainToAnotherDomain"); try { @@ -199,7 +200,7 @@ private void processMoveSubDomainToAnotherDomain(AtlasEntity domain, LOG.info("Moving subdomain {} to Domain {}", domainName, targetDomainQualifiedName); - domainExists(domainName, targetDomainQualifiedName); + domainExists(domainName, targetDomainQualifiedName, guid); if(targetDomainQualifiedName.isEmpty()){ //Moving subDomain to make it Super Domain @@ -356,10 +357,10 @@ private AtlasEntityHeader getParent(AtlasEntity domainEntity) throws AtlasBaseEx return getParent(objectId, PARENT_ATTRIBUTES); } - private void domainExists(String domainName, String parentDomainQualifiedName) throws AtlasBaseException { + private void domainExists(String domainName, String parentDomainQualifiedName,String guid) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("domainExists"); try { - exists(DATA_DOMAIN_ENTITY_TYPE, domainName, parentDomainQualifiedName); + exists(DATA_DOMAIN_ENTITY_TYPE, domainName, parentDomainQualifiedName, guid); } finally { RequestContext.get().endMetricRecord(metricRecorder); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 6f25a487ae8..fdb90b587f9 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -95,7 +95,7 @@ private void processCreateProduct(AtlasEntity entity,AtlasVertex vertex) throws entity.setAttribute(SUPER_DOMAIN_QN_ATTR, superDomainQualifiedName); } - productExists(productName, parentDomainQualifiedName); + productExists(productName, parentDomainQualifiedName, null); createDaapVisibilityPolicy(entity, vertex); @@ -144,7 +144,7 @@ private void processUpdateProduct(AtlasEntity entity, AtlasVertex vertex) throws newSuperDomainQualifiedName = newParentDomainQualifiedName; } - processMoveDataProductToAnotherDomain(entity, currentParentDomainQualifiedName, newParentDomainQualifiedName, vertexQnName, newSuperDomainQualifiedName); + processMoveDataProductToAnotherDomain(entity, currentParentDomainQualifiedName, newParentDomainQualifiedName, vertexQnName, newSuperDomainQualifiedName, storedProduct.getGuid()); updatePolicies(this.updatedPolicyResources, this.context); @@ -159,7 +159,7 @@ private void processUpdateProduct(AtlasEntity entity, AtlasVertex vertex) throws String productNewName = (String) entity.getAttribute(NAME); if (!productCurrentName.equals(productNewName)) { - productExists(productNewName, currentParentDomainQualifiedName); + productExists(productNewName, currentParentDomainQualifiedName, storedProduct.getGuid()); } entity.setAttribute(QUALIFIED_NAME, vertexQnName); } @@ -179,7 +179,8 @@ private void processMoveDataProductToAnotherDomain(AtlasEntity product, String sourceDomainQualifiedName, String targetDomainQualifiedName, String currentDataProductQualifiedName, - String superDomainQualifiedName) throws AtlasBaseException { + String superDomainQualifiedName, + String guid) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("processMoveDataProductToAnotherDomain"); try { @@ -187,7 +188,7 @@ private void processMoveDataProductToAnotherDomain(AtlasEntity product, LOG.info("Moving dataProduct {} to Domain {}", productName, targetDomainQualifiedName); - productExists(productName, targetDomainQualifiedName); + productExists(productName, targetDomainQualifiedName, guid); String updatedQualifiedName; if(StringUtils.isEmpty(sourceDomainQualifiedName)){ @@ -221,11 +222,11 @@ private AtlasEntityHeader getParent(AtlasEntity productEntity) throws AtlasBaseE return getParent(relationshipAttribute, PARENT_ATTRIBUTES); } - private void productExists(String productName, String parentDomainQualifiedName) throws AtlasBaseException { + private void productExists(String productName, String parentDomainQualifiedName, String guid) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("domainExists"); try { - exists(DATA_PRODUCT_ENTITY_TYPE, productName, parentDomainQualifiedName); + exists(DATA_PRODUCT_ENTITY_TYPE, productName, parentDomainQualifiedName, guid); } finally { RequestContext.get().endMetricRecord(metricRecorder); From 30b2eef346a3631af1eed557ebfeae05493c3fbd Mon Sep 17 00:00:00 2001 From: hr2904 Date: Tue, 14 May 2024 18:53:07 +0530 Subject: [PATCH 214/316] Fixed QN naming logic for empty prev - relationships. --- .../datamesh/DataDomainPreProcessor.java | 12 ++++++++++-- .../datamesh/DataProductPreProcessor.java | 3 ++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java index c755bdebf1f..b49a5f8e637 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java @@ -160,7 +160,9 @@ private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws } if (!newParentDomainQualifiedName.equals(currentParentDomainQualifiedName) && entity.hasRelationshipAttribute(PARENT_DOMAIN_REL_TYPE)) { - if(storedDomain.getRelationshipAttribute(PARENT_DOMAIN_REL_TYPE) == null){ + if(storedDomain.getRelationshipAttribute(PARENT_DOMAIN_REL_TYPE) == null && + Objects.isNull(storedDomain.getAttribute(PARENT_DOMAIN_QN_ATTR)) && + StringUtils.isEmpty( (String) storedDomain.getAttribute(PARENT_DOMAIN_QN_ATTR))){ throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Cannot move Super Domain inside another domain"); } @@ -213,7 +215,13 @@ private void processMoveSubDomainToAnotherDomain(AtlasEntity domain, superDomainQualifiedName = updatedQualifiedName ; } else{ - updatedQualifiedName = currentDomainQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); + if(StringUtils.isEmpty(sourceDomainQualifiedName)){ + String[] arr = currentDomainQualifiedName.split("/"); + updatedQualifiedName = targetDomainQualifiedName + "/domain/" + arr[arr.length - 1]; + }else { + updatedQualifiedName = currentDomainQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); + } + domain.setAttribute(QUALIFIED_NAME, updatedQualifiedName); domain.setAttribute(PARENT_DOMAIN_QN_ATTR, targetDomainQualifiedName); domain.setAttribute(SUPER_DOMAIN_QN_ATTR, superDomainQualifiedName); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index fdb90b587f9..9bd386692f8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -192,7 +192,8 @@ private void processMoveDataProductToAnotherDomain(AtlasEntity product, String updatedQualifiedName; if(StringUtils.isEmpty(sourceDomainQualifiedName)){ - updatedQualifiedName = targetDomainQualifiedName + "/" + product.getAttribute(QUALIFIED_NAME); + String[] arr = product.getAttribute(QUALIFIED_NAME).toString().split("/"); + updatedQualifiedName = targetDomainQualifiedName + "/product/" + arr[ arr.length - 1]; } else { updatedQualifiedName = currentDataProductQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); } From b20619b45774689dee682ad6e6f8228a27c3405d Mon Sep 17 00:00:00 2001 From: hr2904 Date: Fri, 17 May 2024 11:55:13 +0530 Subject: [PATCH 215/316] Fixed PR comments --- .../datamesh/DataDomainPreProcessor.java | 26 ++++++++----------- .../datamesh/DataProductPreProcessor.java | 16 +++++------- 2 files changed, 18 insertions(+), 24 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java index b49a5f8e637..a8fe85e7725 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java @@ -97,6 +97,14 @@ private void processCreateDomain(AtlasEntity entity) throws AtlasBaseException { if(parentDomainObject != null ){ parentDomain = retrieverNoRelation.getEntityVertex(parentDomainObject); parentDomainQualifiedName = parentDomain.getProperty(QUALIFIED_NAME, String.class); + if(StringUtils.isNotEmpty(parentDomainQualifiedName)) { + entity.setAttribute(PARENT_DOMAIN_QN_ATTR, parentDomainQualifiedName); + String superDomainQualifiedName = parentDomain.getProperty(SUPER_DOMAIN_QN_ATTR, String.class); + if(StringUtils.isEmpty(parentDomain.getProperty(SUPER_DOMAIN_QN_ATTR, String.class))) { + superDomainQualifiedName = parentDomainQualifiedName; + } + entity.setAttribute(SUPER_DOMAIN_QN_ATTR, superDomainQualifiedName); + } } else { entity.removeAttribute(PARENT_DOMAIN_QN_ATTR); entity.removeAttribute(SUPER_DOMAIN_QN_ATTR); @@ -104,17 +112,6 @@ private void processCreateDomain(AtlasEntity entity) throws AtlasBaseException { entity.setAttribute(QUALIFIED_NAME, createQualifiedName(parentDomainQualifiedName)); - if(StringUtils.isNotEmpty(parentDomainQualifiedName)) { - entity.setAttribute(PARENT_DOMAIN_QN_ATTR, parentDomainQualifiedName); - String superDomainQualifiedName = ""; - if(StringUtils.isEmpty(parentDomain.getProperty(SUPER_DOMAIN_QN_ATTR, String.class))){ - superDomainQualifiedName = parentDomainQualifiedName; - } - else { - superDomainQualifiedName = parentDomain.getProperty(SUPER_DOMAIN_QN_ATTR, String.class); - } - entity.setAttribute(SUPER_DOMAIN_QN_ATTR, superDomainQualifiedName); - } entity.setCustomAttributes(customAttributes); @@ -169,7 +166,7 @@ private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws //Auth check isAuthorized(currentParentDomainHeader, newParentDomainHeader); - processMoveSubDomainToAnotherDomain(entity, vertex, currentParentDomainQualifiedName, newParentDomainQualifiedName, vertexQnName, newSuperDomainQualifiedName, storedDomain.getGuid()); + processMoveSubDomainToAnotherDomain(entity, vertex, currentParentDomainQualifiedName, newParentDomainQualifiedName, vertexQnName, newSuperDomainQualifiedName); } else { String domainCurrentName = vertex.getProperty(NAME, String.class); @@ -192,8 +189,7 @@ private void processMoveSubDomainToAnotherDomain(AtlasEntity domain, String sourceDomainQualifiedName, String targetDomainQualifiedName, String currentDomainQualifiedName, - String superDomainQualifiedName, - String guid) throws AtlasBaseException { + String superDomainQualifiedName) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("processMoveSubDomainToAnotherDomain"); try { @@ -202,7 +198,7 @@ private void processMoveSubDomainToAnotherDomain(AtlasEntity domain, LOG.info("Moving subdomain {} to Domain {}", domainName, targetDomainQualifiedName); - domainExists(domainName, targetDomainQualifiedName, guid); + domainExists(domainName, targetDomainQualifiedName, domain.getGuid()); if(targetDomainQualifiedName.isEmpty()){ //Moving subDomain to make it Super Domain diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 9bd386692f8..e09cf49609d 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -79,6 +79,7 @@ private void processCreateProduct(AtlasEntity entity,AtlasVertex vertex) throws AtlasObjectId parentDomainObject = (AtlasObjectId) entity.getRelationshipAttribute(DATA_DOMAIN_REL_TYPE); String productName = (String) entity.getAttribute(NAME); String parentDomainQualifiedName = ""; + entity.setAttribute(QUALIFIED_NAME, createQualifiedName(parentDomainQualifiedName)); if (parentDomainObject == null) { entity.removeAttribute(PARENT_DOMAIN_QN_ATTR); @@ -87,7 +88,6 @@ private void processCreateProduct(AtlasEntity entity,AtlasVertex vertex) throws AtlasVertex parentDomain = retrieverNoRelation.getEntityVertex(parentDomainObject); parentDomainQualifiedName = parentDomain.getProperty(QUALIFIED_NAME, String.class); - entity.setAttribute(QUALIFIED_NAME, createQualifiedName(parentDomainQualifiedName)); entity.setAttribute(PARENT_DOMAIN_QN_ATTR, parentDomainQualifiedName); @@ -144,16 +144,15 @@ private void processUpdateProduct(AtlasEntity entity, AtlasVertex vertex) throws newSuperDomainQualifiedName = newParentDomainQualifiedName; } - processMoveDataProductToAnotherDomain(entity, currentParentDomainQualifiedName, newParentDomainQualifiedName, vertexQnName, newSuperDomainQualifiedName, storedProduct.getGuid()); + processMoveDataProductToAnotherDomain(entity, currentParentDomainQualifiedName, newParentDomainQualifiedName, vertexQnName, newSuperDomainQualifiedName); updatePolicies(this.updatedPolicyResources, this.context); } else { entity.removeAttribute(PARENT_DOMAIN_QN_ATTR); entity.removeAttribute(SUPER_DOMAIN_QN_ATTR); - currentParentDomainQualifiedName = ""; - if (entity.getRelationshipAttributes() != null) { - entity.getRelationshipAttributes().remove(DATA_PRODUCT_REL_TYPE); + if(Objects.isNull(currentParentDomainQualifiedName)) { + currentParentDomainQualifiedName = ""; } String productCurrentName = vertex.getProperty(NAME, String.class); String productNewName = (String) entity.getAttribute(NAME); @@ -179,8 +178,7 @@ private void processMoveDataProductToAnotherDomain(AtlasEntity product, String sourceDomainQualifiedName, String targetDomainQualifiedName, String currentDataProductQualifiedName, - String superDomainQualifiedName, - String guid) throws AtlasBaseException { + String superDomainQualifiedName) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("processMoveDataProductToAnotherDomain"); try { @@ -188,7 +186,7 @@ private void processMoveDataProductToAnotherDomain(AtlasEntity product, LOG.info("Moving dataProduct {} to Domain {}", productName, targetDomainQualifiedName); - productExists(productName, targetDomainQualifiedName, guid); + productExists(productName, targetDomainQualifiedName, product.getGuid()); String updatedQualifiedName; if(StringUtils.isEmpty(sourceDomainQualifiedName)){ @@ -224,7 +222,7 @@ private AtlasEntityHeader getParent(AtlasEntity productEntity) throws AtlasBaseE } private void productExists(String productName, String parentDomainQualifiedName, String guid) throws AtlasBaseException { - AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("domainExists"); + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("productExists"); try { exists(DATA_PRODUCT_ENTITY_TYPE, productName, parentDomainQualifiedName, guid); From 62d0dd4c136daf9c0805bdcb3917d04e306c6575 Mon Sep 17 00:00:00 2001 From: suraj5077 Date: Fri, 17 May 2024 12:32:36 +0530 Subject: [PATCH 216/316] fix: base entity finish time --- .../org/apache/atlas/discovery/EntityLineageService.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index 0b079d9e5f2..de939eac550 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -302,8 +302,7 @@ private AtlasLineageOnDemandInfo getLineageInfoOnDemand(String guid, AtlasLineag if (direction == AtlasLineageOnDemandInfo.LineageDirection.OUTPUT || direction == AtlasLineageOnDemandInfo.LineageDirection.BOTH) traverseEdgesOnDemand(datasetVertex, false, depth, level, new HashSet<>(), atlasLineageOnDemandContext, ret, guid, outputEntitiesTraversed, traversalOrder); AtlasEntityHeader baseEntityHeader = entityRetriever.toAtlasEntityHeader(datasetVertex, atlasLineageOnDemandContext.getAttributes()); - baseEntityHeader.setDepth(level); - baseEntityHeader.setTraversalOrder(0); + setGraphTraversalMetadata(level, traversalOrder, baseEntityHeader); ret.getGuidEntityMap().put(guid, baseEntityHeader); } else { AtlasVertex processVertex = AtlasGraphUtilsV2.findByGuid(this.graph, guid); @@ -321,6 +320,12 @@ private AtlasLineageOnDemandInfo getLineageInfoOnDemand(String guid, AtlasLineag return ret; } + private static void setGraphTraversalMetadata(int level, AtomicInteger traversalOrder, AtlasEntityHeader baseEntityHeader) { + baseEntityHeader.setDepth(level); + baseEntityHeader.setTraversalOrder(0); + baseEntityHeader.setFinishTime(traversalOrder.get()); + } + private void traverseEdgesOnDemand(Iterator processEdges, boolean isInput, int depth, int level, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, AtlasVertex processVertex, String baseGuid, AtomicInteger entitiesTraversed, AtomicInteger traversalOrder) throws AtlasBaseException { AtlasLineageOnDemandInfo.LineageDirection direction = isInput ? AtlasLineageOnDemandInfo.LineageDirection.INPUT : AtlasLineageOnDemandInfo.LineageDirection.OUTPUT; int nextLevel = isInput ? level - 1: level + 1; From 5bad6750f2cb96cd72e403fa0e85d91b7a6dc2fc Mon Sep 17 00:00:00 2001 From: hr2904 Date: Fri, 17 May 2024 12:57:05 +0530 Subject: [PATCH 217/316] Fixed PR comments #2 --- .../v2/preprocessor/datamesh/AbstractDomainPreProcessor.java | 2 +- .../graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java index f39279fc537..f62740bc67f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/AbstractDomainPreProcessor.java @@ -226,7 +226,7 @@ protected void exists(String assetType, String assetName, String parentDomainQua mustClauseList.add(mapOf("term", mapOf("__state", "ACTIVE"))); mustClauseList.add(mapOf("term", mapOf("name.keyword", assetName))); List> mustNotClauseList = new ArrayList(); - if(Objects.nonNull(guid) && StringUtils.isNotEmpty(guid)){ + if(StringUtils.isNotEmpty(guid)){ mustNotClauseList.add(mapOf("term", mapOf("__guid", guid))); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java index a8fe85e7725..e0195edb3a1 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java @@ -158,7 +158,6 @@ private void processUpdateDomain(AtlasEntity entity, AtlasVertex vertex) throws if (!newParentDomainQualifiedName.equals(currentParentDomainQualifiedName) && entity.hasRelationshipAttribute(PARENT_DOMAIN_REL_TYPE)) { if(storedDomain.getRelationshipAttribute(PARENT_DOMAIN_REL_TYPE) == null && - Objects.isNull(storedDomain.getAttribute(PARENT_DOMAIN_QN_ATTR)) && StringUtils.isEmpty( (String) storedDomain.getAttribute(PARENT_DOMAIN_QN_ATTR))){ throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Cannot move Super Domain inside another domain"); } From 3fa372c25f74b5b6a2639fc9730cba7c7b30e9a6 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Fri, 17 May 2024 13:08:17 +0530 Subject: [PATCH 218/316] Fixed a PR commit #3 --- .../v2/preprocessor/datamesh/DataProductPreProcessor.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index e09cf49609d..6a3a81f8714 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -151,9 +151,6 @@ private void processUpdateProduct(AtlasEntity entity, AtlasVertex vertex) throws } else { entity.removeAttribute(PARENT_DOMAIN_QN_ATTR); entity.removeAttribute(SUPER_DOMAIN_QN_ATTR); - if(Objects.isNull(currentParentDomainQualifiedName)) { - currentParentDomainQualifiedName = ""; - } String productCurrentName = vertex.getProperty(NAME, String.class); String productNewName = (String) entity.getAttribute(NAME); From 414720583f0249b0b6be8a8ae200c9143ae8dbf7 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Fri, 17 May 2024 13:39:19 +0530 Subject: [PATCH 219/316] Simplified logic for new QN generation incase of updation of domain and product. --- .../graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java | 3 +-- .../v2/preprocessor/datamesh/DataProductPreProcessor.java | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java index e0195edb3a1..c056b2ad68f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java @@ -211,8 +211,7 @@ private void processMoveSubDomainToAnotherDomain(AtlasEntity domain, } else{ if(StringUtils.isEmpty(sourceDomainQualifiedName)){ - String[] arr = currentDomainQualifiedName.split("/"); - updatedQualifiedName = targetDomainQualifiedName + "/domain/" + arr[arr.length - 1]; + updatedQualifiedName = createQualifiedName(targetDomainQualifiedName); }else { updatedQualifiedName = currentDomainQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 6a3a81f8714..5157f0b0657 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -187,8 +187,7 @@ private void processMoveDataProductToAnotherDomain(AtlasEntity product, String updatedQualifiedName; if(StringUtils.isEmpty(sourceDomainQualifiedName)){ - String[] arr = product.getAttribute(QUALIFIED_NAME).toString().split("/"); - updatedQualifiedName = targetDomainQualifiedName + "/product/" + arr[ arr.length - 1]; + updatedQualifiedName = createQualifiedName(targetDomainQualifiedName); } else { updatedQualifiedName = currentDataProductQualifiedName.replace(sourceDomainQualifiedName, targetDomainQualifiedName); } From f75a2e78be28b6fcf2d294a5c2559465e4f83d59 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Fri, 17 May 2024 14:01:56 +0530 Subject: [PATCH 220/316] Fixed a logic error --- .../v2/preprocessor/datamesh/DataProductPreProcessor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 5157f0b0657..b5e7c1aa2cd 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -79,7 +79,6 @@ private void processCreateProduct(AtlasEntity entity,AtlasVertex vertex) throws AtlasObjectId parentDomainObject = (AtlasObjectId) entity.getRelationshipAttribute(DATA_DOMAIN_REL_TYPE); String productName = (String) entity.getAttribute(NAME); String parentDomainQualifiedName = ""; - entity.setAttribute(QUALIFIED_NAME, createQualifiedName(parentDomainQualifiedName)); if (parentDomainObject == null) { entity.removeAttribute(PARENT_DOMAIN_QN_ATTR); @@ -95,6 +94,8 @@ private void processCreateProduct(AtlasEntity entity,AtlasVertex vertex) throws entity.setAttribute(SUPER_DOMAIN_QN_ATTR, superDomainQualifiedName); } + entity.setAttribute(QUALIFIED_NAME, createQualifiedName(parentDomainQualifiedName)); + productExists(productName, parentDomainQualifiedName, null); createDaapVisibilityPolicy(entity, vertex); From 15dc801afb5384edc828a660e07ad4a49deeca16 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Fri, 17 May 2024 18:17:32 +0530 Subject: [PATCH 221/316] DG - 1370 : Fixed : if superDomain does not exists, parent is the superDomain --- .../v2/preprocessor/datamesh/DataProductPreProcessor.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index b5e7c1aa2cd..1927e79caf6 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -91,6 +91,9 @@ private void processCreateProduct(AtlasEntity entity,AtlasVertex vertex) throws entity.setAttribute(PARENT_DOMAIN_QN_ATTR, parentDomainQualifiedName); String superDomainQualifiedName = parentDomain.getProperty(SUPER_DOMAIN_QN_ATTR, String.class); + if(StringUtils.isEmpty(superDomainQualifiedName)) { + superDomainQualifiedName = parentDomainQualifiedName; + } entity.setAttribute(SUPER_DOMAIN_QN_ATTR, superDomainQualifiedName); } From 0df9897755220a423bf9fcc0a0549c947f08e333 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Tue, 21 May 2024 18:05:21 +0530 Subject: [PATCH 222/316] feat: implement cleanup tag task --- .../repository/graphdb/AtlasIndexQuery.java | 3 + .../janus/AtlasElasticsearchQuery.java | 28 ++++ .../graphdb/janus/AtlasJanusIndexQuery.java | 6 + .../discovery/AtlasDiscoveryService.java | 9 + .../discovery/EntityDiscoveryService.java | 45 ++++- .../converters/AtlasInstanceConverter.java | 10 ++ .../atlas/repository/graph/GraphHelper.java | 12 +- .../store/graph/v2/AtlasEntityStoreV2.java | 6 +- .../store/graph/v2/EntityGraphMapper.java | 154 ++++++++++++------ .../graph/v2/bulkimport/MigrationImport.java | 2 +- .../ClassificationPropagateTaskFactory.java | 7 + .../tasks/ClassificationPropagationTasks.java | 14 ++ .../graph/v2/tasks/ClassificationTask.java | 9 +- .../apache/atlas/tasks/AtlasTaskService.java | 2 +- 14 files changed, 242 insertions(+), 65 deletions(-) diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java index 09fa0d6d060..0da85b796ca 100644 --- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java +++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java @@ -22,6 +22,7 @@ import org.apache.atlas.model.discovery.SearchParams; import org.apache.tinkerpop.gremlin.process.traversal.Order; +import java.io.IOException; import java.util.Iterator; import java.util.List; import java.util.Map; @@ -43,6 +44,8 @@ public interface AtlasIndexQuery { DirectIndexQueryResult vertices(SearchParams searchParams) throws AtlasBaseException ; + int getNumVertices(SearchParams searchParams) throws AtlasBaseException, IOException; + /** * Gets the query results form index * diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java index c3030291460..2dddd1ef155 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java @@ -128,6 +128,34 @@ private Iterator> runQuery(SearchReques return result; } + public int getNumVertices(SearchParams searchParams) throws AtlasBaseException, IOException { + // Make a count query + String query = searchParams.getQuery(); + HttpEntity entity = new NStringEntity(query, ContentType.APPLICATION_JSON); + String endPoint = index + "/_count"; + Request request = new Request("GET", endPoint); + request.setEntity(entity); + Response response; + try { + response = lowLevelRestClient.performRequest(request); + } catch (ResponseException rex) { + if (rex.getResponse().getStatusLine().getStatusCode() == 404) { + LOG.warn(String.format("ES index with name %s not found", index)); + throw new AtlasBaseException(INDEX_NOT_FOUND, index); + } else { + throw new AtlasBaseException(rex); + } + } catch (IOException e) { + throw new AtlasBaseException(AtlasErrorCode.INDEX_SEARCH_FAILED, e.getMessage()); + } + return getVertexCountFromResponse(EntityUtils.toString(response.getEntity())); + + } + int getVertexCountFromResponse(String responseString) throws IOException { + Map responseMap = AtlasType.fromJson(responseString, Map.class); + return (Integer) responseMap.get("count"); + } + private DirectIndexQueryResult runQueryWithLowLevelClient(SearchParams searchParams) throws AtlasBaseException { DirectIndexQueryResult result = null; diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusIndexQuery.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusIndexQuery.java index d4417e8f6e3..ba680361d08 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusIndexQuery.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusIndexQuery.java @@ -17,6 +17,7 @@ */ package org.apache.atlas.repository.graphdb.janus; +import java.io.IOException; import java.util.*; import com.google.common.base.Preconditions; @@ -49,6 +50,11 @@ public DirectIndexQueryResult vertices(SearchP return null; } + @Override + public int getNumVertices(SearchParams searchParams) throws AtlasBaseException, IOException { + return 0; + } + @Override public Map directIndexQuery(String query) throws AtlasBaseException { return null; diff --git a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java index dc567be4fe3..e898190d272 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java @@ -29,6 +29,7 @@ import org.apache.atlas.model.profile.AtlasUserSavedSearch; import org.apache.atlas.model.searchlog.SearchLogSearchParams; import org.apache.atlas.model.searchlog.SearchLogSearchResult; +import org.apache.atlas.repository.graphdb.DirectIndexQueryResult; import java.util.List; @@ -158,6 +159,14 @@ AtlasSearchResult searchUsingBasicQuery(String query, String type, String classi */ AtlasSearchResult directIndexSearch(SearchParams searchParams) throws AtlasBaseException; + /** + * Search for direct ES query + * @param searchParams Search criteria + * @return Matching entities + * @throws AtlasBaseException + */ + DirectIndexQueryResult directIndexSearch(SearchParams searchParams, boolean returnVertices) throws AtlasBaseException; + /** * Search for direct ES query on search logs index * @param searchParams Search criteria diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index 66e8d589091..fb768cd6b89 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -46,7 +46,6 @@ import org.apache.atlas.repository.userprofile.UserProfileService; import org.apache.atlas.repository.util.AccessControlUtils; import org.apache.atlas.searchlog.ESSearchLogger; -import org.apache.atlas.service.FeatureFlagStore; import org.apache.atlas.stats.StatsClient; import org.apache.atlas.type.*; import org.apache.atlas.type.AtlasBuiltInTypes.AtlasObjectIdType; @@ -70,6 +69,7 @@ import javax.inject.Inject; import javax.script.ScriptEngine; import javax.script.ScriptException; +import java.io.IOException; import java.util.*; import java.util.stream.Collectors; @@ -1013,6 +1013,49 @@ public AtlasSearchResult directIndexSearch(SearchParams searchParams) throws Atl return ret; } + public int getQueryResponseCount(SearchParams searchParams) throws AtlasBaseException, IOException { + IndexSearchParams params = (IndexSearchParams) searchParams; + String indexName = getIndexName(params); + AtlasIndexQuery indexQuery = graph.elasticsearchQuery(indexName); + return indexQuery.getNumVertices(searchParams); + + } + + @Override + public DirectIndexQueryResult directIndexSearch(SearchParams searchParams, boolean returnVertices) throws AtlasBaseException { + IndexSearchParams params = (IndexSearchParams) searchParams; + RequestContext.get().setRelationAttrsForSearch(params.getRelationAttributes()); + RequestContext.get().setAllowDeletedRelationsIndexsearch(params.isAllowDeletedRelations()); + + DirectIndexQueryResult ret = null; + AtlasIndexQuery indexQuery; + + Set resultAttributes = new HashSet<>(); + if (CollectionUtils.isNotEmpty(searchParams.getAttributes())) { + resultAttributes.addAll(searchParams.getAttributes()); + } + + try { + if(LOG.isDebugEnabled()){ + LOG.debug("Performing ES search for the params ({})", searchParams); + } + + String indexName = getIndexName(params); + + indexQuery = graph.elasticsearchQuery(indexName); + AtlasPerfMetrics.MetricRecorder elasticSearchQueryMetric = RequestContext.get().startMetricRecord("elasticSearchQuery"); + ret = indexQuery.vertices(searchParams); + if (ret == null) { + return null; + } + RequestContext.get().endMetricRecord(elasticSearchQueryMetric); + } catch (Exception e) { + LOG.error("Error while performing direct search for the params ({}), {}", searchParams, e.getMessage()); + throw e; + } + return ret; + } + @Override public SearchLogSearchResult searchLogs(SearchLogSearchParams searchParams) throws AtlasBaseException { SearchLogSearchResult ret = new SearchLogSearchResult(); diff --git a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java index c1aef7b5098..5c5c0811392 100644 --- a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java +++ b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java @@ -322,6 +322,16 @@ public AtlasEntity getAndCacheEntity(String guid, boolean ignoreRelationshipAttr return entity; } + public AtlasEntity getEntity(String guid,boolean ignoreRelationshipAttributes) throws AtlasBaseException { + AtlasEntity entity = null; + if (ignoreRelationshipAttributes) { + entity = entityGraphRetrieverIgnoreRelationshipAttrs.toAtlasEntity(guid); + } else { + entity = entityGraphRetriever.toAtlasEntity(guid); + } + return entity; + } + public AtlasEntityWithExtInfo getAndCacheEntityExtInfo(String guid) throws AtlasBaseException { RequestContext context = RequestContext.get(); diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index 7907be09d40..d86c74414cf 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -908,18 +908,18 @@ public static boolean propagatedClassificationAttachedToVertex(AtlasVertex class } public static List getClassificationEdges(AtlasVertex entityVertex) { - return getClassificationEdges(entityVertex, false); + return getClassificationEdges(entityVertex, false, null); } public static List getPropagatedClassificationEdges(AtlasVertex entityVertex) { - return getClassificationEdges(entityVertex, true); + return getClassificationEdges(entityVertex, true, null); } public static List getAllClassificationEdges(AtlasVertex entityVertex) { - return getClassificationEdges(entityVertex, null); + return getClassificationEdges(entityVertex, null, null); } - public static List getClassificationEdges(AtlasVertex entityVertex, Boolean propagated) { + public static List getClassificationEdges(AtlasVertex entityVertex, Boolean propagated, String typeName) { List ret = new ArrayList<>(); AtlasVertexQuery query = entityVertex.query().direction(AtlasEdgeDirection.OUT).label(CLASSIFICATION_LABEL); @@ -927,6 +927,10 @@ public static List getClassificationEdges(AtlasVertex entityVertex, B query = query.has(CLASSIFICATION_EDGE_IS_PROPAGATED_PROPERTY_KEY, propagated); } + if (StringUtils.isNotEmpty(typeName)) { + query = query.has(CLASSIFICATION_EDGE_NAME_PROPERTY_KEY, typeName); + } + Iterable edges = query.edges(); if (edges != null) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 8d793e47405..2d6d28c0a72 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -935,11 +935,7 @@ public void repairClassificationMappings(final String guid) throws AtlasBaseExce throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guid); } - AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeaderWithClassifications(entityVertex); - - AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE_CLASSIFICATION, entityHeader), "repair classification mappings: guid=", guid); - - entityGraphMapper.repairClassificationMappings(entityHeader, entityVertex); + entityGraphMapper.repairClassificationMappings(entityVertex); if (LOG.isDebugEnabled()) { LOG.debug("<== repairClassificationMappings({})", guid); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index a0bb952973f..85286282c96 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -24,10 +24,12 @@ import org.apache.atlas.authorize.AtlasAuthorizationUtils; import org.apache.atlas.authorize.AtlasEntityAccessRequest; import org.apache.atlas.authorize.AtlasPrivilege; +import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.exception.EntityNotFoundException; import org.apache.atlas.model.TimeBoundary; import org.apache.atlas.model.TypeCategory; +import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.instance.AtlasClassification; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntityHeader; @@ -53,6 +55,8 @@ import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.graphdb.DirectIndexQueryResult; +import org.apache.atlas.repository.graphdb.AtlasIndexQuery; import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; import org.apache.atlas.repository.store.graph.EntityGraphDiscoveryContext; import org.apache.atlas.repository.store.graph.v1.DeleteHandlerDelegate; @@ -82,22 +86,11 @@ import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; -import javax.inject.Inject; -import java.util.ArrayList; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.NoSuchElementException; -import java.util.Objects; -import java.util.Set; -import java.util.UUID; -import java.util.Date; +import java.io.IOException; +import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -203,13 +196,15 @@ public class EntityGraphMapper { private final EntityGraphRetriever entityRetriever; private final IFullTextMapper fullTextMapperV2; private final TaskManagement taskManagement; + + private final EntityDiscoveryService discoveryService; private final TransactionInterceptHelper transactionInterceptHelper; - @Inject + @Autowired public EntityGraphMapper(DeleteHandlerDelegate deleteDelegate, RestoreHandlerV1 restoreHandlerV1, AtlasTypeRegistry typeRegistry, AtlasGraph graph, AtlasRelationshipStore relationshipStore, IAtlasEntityChangeNotifier entityChangeNotifier, AtlasInstanceConverter instanceConverter, IFullTextMapper fullTextMapperV2, - TaskManagement taskManagement, TransactionInterceptHelper transactionInterceptHelper) { + TaskManagement taskManagement, TransactionInterceptHelper transactionInterceptHelper,@Lazy EntityDiscoveryService discoveryService) { this.restoreHandlerV1 = restoreHandlerV1; this.graphHelper = new GraphHelper(graph); this.deleteDelegate = deleteDelegate; @@ -222,6 +217,7 @@ public EntityGraphMapper(DeleteHandlerDelegate deleteDelegate, RestoreHandlerV1 this.fullTextMapperV2 = fullTextMapperV2; this.taskManagement = taskManagement; this.transactionInterceptHelper = transactionInterceptHelper; + this.discoveryService = discoveryService; } @VisibleForTesting @@ -2980,35 +2976,93 @@ private void updateInConsistentOwnedMapVertices(AttributeMutationContext ctx, At } } - public void repairClassificationMappings(AtlasEntityHeader entityHeader, AtlasVertex entityVertex) throws AtlasBaseException { - if (entityHeader.getClassifications() != null) { - List classifications = entityHeader.getClassifications(); - String propagatedClassificationName = null; - String classificationNames = null; - String classificationText = null; - List propagatedClassificationNamesList = new ArrayList<>(); - List classificationNamesList = new ArrayList<>(); - for (AtlasClassification classification : classifications) { - String typeName = classification.getTypeName(); - classificationText = fullTextMapperV2.getClassificationTextForEntity(new AtlasEntity(entityHeader)); - if(classification.isPropagate() && !classification.getEntityGuid().equals(entityHeader.getGuid())) { - //Add to propagatedClassificationNamesList - propagatedClassificationNamesList.add(typeName); - //Create delimiter separated string for propagatedClassificationNames - if (StringUtils.isEmpty(propagatedClassificationName)) { - propagatedClassificationName = CLASSIFICATION_NAME_DELIMITER + typeName + CLASSIFICATION_NAME_DELIMITER; - } else { - propagatedClassificationName = propagatedClassificationName + typeName + CLASSIFICATION_NAME_DELIMITER; + public void cleanUpClassificationPropagation(String classificationName) throws AtlasBaseException, IOException { + IndexSearchParams searchParams = new IndexSearchParams(); + int batchSize = 100; + DirectIndexQueryResult indexQueryResult; + Map dsl = buildMap(classificationName); + searchParams.setDsl(dsl); + int totalCount = discoveryService.getQueryResponseCount(searchParams); + LOG.info("Total entities found for classification {} are {}", classificationName, totalCount); + int counter = 0; + dsl.put("from", 0); + dsl.put("size", batchSize); + searchParams.setDsl(dsl); + while (counter <= totalCount) { + try { + indexQueryResult = discoveryService.directIndexSearch(searchParams, true); + if (indexQueryResult == null) { + if (LOG.isDebugEnabled()) { + LOG.debug("No entities found for classification {}", classificationName); } - } else { - //Add to classificationNamesList - classificationNamesList.add(typeName); - //Create delimiter separated string for classificationNames - if (StringUtils.isEmpty(classificationNames)) { - classificationNames = CLASSIFICATION_NAME_DELIMITER + typeName + CLASSIFICATION_NAME_DELIMITER; - } else { - classificationNames = classificationNames + typeName + CLASSIFICATION_NAME_DELIMITER; + return; + } + Iterator results = indexQueryResult.getIterator(); + if (!results.hasNext()) { + LOG.info("No entities found for classification {}", classificationName); + return; + } + while (results.hasNext()) { + AtlasVertex vertex = results.next().getVertex(); + String guid = GraphHelper.getGuid(vertex); + GraphTransactionInterceptor.lockObjectAndReleasePostCommit(guid); + List deletedClassifications = new ArrayList<>(); + List classificationEdges = GraphHelper.getClassificationEdges(vertex, null, classificationName); + + for (AtlasEdge edge : classificationEdges) { + AtlasClassification classification = entityRetriever.toAtlasClassification(edge.getInVertex()); + deletedClassifications.add(classification); + deleteDelegate.getHandler().deleteEdgeReference(edge, TypeCategory.CLASSIFICATION, false, true, null, vertex); } + + AtlasEntity entity = repairClassificationMappings(vertex); + + entityChangeNotifier.onClassificationDeletedFromEntity(entity, deletedClassifications); + + counter++; + } + } catch (AtlasBaseException e) { + throw new AtlasBaseException(AtlasErrorCode.INDEX_SEARCH_FAILED, e); + } finally { + transactionInterceptHelper.intercept(); + LOG.info("Processed cleaning up {} entities", counter); + } + } + } + + private Map getMap(String key, Object value) { + Map map = new HashMap<>(); + map.put(key, value); + return map; + } + + public Map buildMap(String classificationName) { + Map dsl = new HashMap<>(); + Map query = new HashMap<>(); + List shouldClauses = new ArrayList<>(); + Map terms1 = getMap("terms", getMap("__traitNames", Collections.singletonList(classificationName))); + Map terms2 = getMap("terms", getMap("__propagatedTraitNames", Collections.singletonList(classificationName))); + shouldClauses.add(terms1); + shouldClauses.add(terms2); + query.put("bool", getMap("should", shouldClauses)); + dsl.put("query", query); + return dsl; + } + public AtlasEntity repairClassificationMappings(AtlasVertex entityVertex) throws AtlasBaseException { + String guid = GraphHelper.getGuid(entityVertex); + AtlasEntity entity = instanceConverter.getEntity(guid, ENTITY_CHANGE_NOTIFY_IGNORE_RELATIONSHIP_ATTRIBUTES); + + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE_CLASSIFICATION, new AtlasEntityHeader(entity)), "repair classification mappings: guid=", guid); + + if (entity.getClassifications() != null) { + List classifications = entity.getClassifications(); + List classificationNames = new ArrayList<>(); + List propagatedClassificationNames = new ArrayList<>(); + for (AtlasClassification classification : classifications) { + if (isPropagatedClassification(classification, guid)) { + propagatedClassificationNames.add(classification.getTypeName()); + } else { + classificationNames.add(classification.getTypeName()); } } //Delete array/set properties first @@ -3017,20 +3071,20 @@ public void repairClassificationMappings(AtlasEntityHeader entityHeader, AtlasVe //Update classificationNames and propagatedClassificationNames in entityVertex - AtlasGraphUtilsV2.setProperty(entityVertex, CLASSIFICATION_NAMES_KEY, classificationNames); - AtlasGraphUtilsV2.setProperty(entityVertex, CLASSIFICATION_TEXT_KEY, classificationText); - AtlasGraphUtilsV2.setProperty(entityVertex, PROPAGATED_CLASSIFICATION_NAMES_KEY, propagatedClassificationName); + entityVertex.setProperty(CLASSIFICATION_NAMES_KEY, getDelimitedClassificationNames(classificationNames)); + entityVertex.setProperty(PROPAGATED_CLASSIFICATION_NAMES_KEY, getDelimitedClassificationNames(propagatedClassificationNames)); + entityVertex.setProperty(CLASSIFICATION_TEXT_KEY, fullTextMapperV2.getClassificationTextForEntity(entity)); // Make classificationNames unique list as it is of type SET - classificationNamesList = classificationNamesList.stream().distinct().collect(Collectors.toList()); + classificationNames = classificationNames.stream().distinct().collect(Collectors.toList()); //Update classificationNames and propagatedClassificationNames in entityHeader - for(String classificationName : classificationNamesList) { + for(String classificationName : classificationNames) { AtlasGraphUtilsV2.addEncodedProperty(entityVertex, TRAIT_NAMES_PROPERTY_KEY, classificationName); } - for (String classificationName : propagatedClassificationNamesList) { - //AtlasGraphUtilsV2.addProperty(entityVertex, PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationName); + for (String classificationName : propagatedClassificationNames) { entityVertex.addListProperty(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationName); } } + return entity; } public void addClassifications(final EntityMutationContext context, String guid, List classifications) throws AtlasBaseException { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/MigrationImport.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/MigrationImport.java index 0cc7c4a318b..df13282b614 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/MigrationImport.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/MigrationImport.java @@ -126,7 +126,7 @@ private AtlasEntityStoreV2 createEntityStore(AtlasGraph graph, AtlasTypeRegistry AtlasRelationshipStore relationshipStore = new AtlasRelationshipStoreV2(graph, typeRegistry, deleteDelegate, entityChangeNotifier); EntityGraphMapper entityGraphMapper = new EntityGraphMapper(deleteDelegate, restoreHandlerV1, typeRegistry, - graph, relationshipStore, entityChangeNotifier, getInstanceConverter(graph), fullTextMapperV2, null, null); + graph, relationshipStore, entityChangeNotifier, getInstanceConverter(graph), fullTextMapperV2, null, null, null); AtlasRelationshipStoreV2 atlasRelationshipStoreV2 = new AtlasRelationshipStoreV2(graph, typeRegistry, deleteDelegate, entityChangeNotifier); return new AtlasEntityStoreV2(graph, deleteDelegate, restoreHandlerV1, typeRegistry, entityChangeNotifier, entityGraphMapper, null, atlasRelationshipStoreV2, null); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationPropagateTaskFactory.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationPropagateTaskFactory.java index ca32e234087..65ac992b17d 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationPropagateTaskFactory.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationPropagateTaskFactory.java @@ -54,6 +54,8 @@ public class ClassificationPropagateTaskFactory implements TaskFactory { public static final String CLASSIFICATION_PROPAGATION_RELATIONSHIP_UPDATE = "CLASSIFICATION_PROPAGATION_RELATIONSHIP_UPDATE"; + public static final String CLEANUP_CLASSIFICATION_PROPAGATION = "CLEANUP_CLASSIFICATION_PROPAGATION"; + public static final List supportedTypes = new ArrayList() {{ @@ -63,6 +65,7 @@ public class ClassificationPropagateTaskFactory implements TaskFactory { add(CLASSIFICATION_ONLY_PROPAGATION_DELETE_ON_HARD_DELETE); add(CLASSIFICATION_REFRESH_PROPAGATION); add(CLASSIFICATION_PROPAGATION_RELATIONSHIP_UPDATE); + add(CLEANUP_CLASSIFICATION_PROPAGATION); }}; @@ -102,6 +105,10 @@ public org.apache.atlas.tasks.AbstractTask create(AtlasTask task) { case CLASSIFICATION_PROPAGATION_RELATIONSHIP_UPDATE: return new ClassificationPropagationTasks.UpdateRelationship(task, graph, entityGraphMapper, deleteDelegate, relationshipStore); + case CLEANUP_CLASSIFICATION_PROPAGATION: + return new ClassificationPropagationTasks.CleanUpClassificationPropagation(task, graph, entityGraphMapper, deleteDelegate, relationshipStore); + + default: LOG.warn("Type: {} - {} not found!. The task will be ignored.", taskType, taskGuid); return null; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationPropagationTasks.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationPropagationTasks.java index 69abc3aafc4..a3a3e98633b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationPropagationTasks.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationPropagationTasks.java @@ -26,6 +26,7 @@ import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; import org.apache.atlas.type.AtlasType; +import java.io.IOException; import java.util.Map; import java.util.Set; @@ -121,4 +122,17 @@ protected void run(Map parameters) throws AtlasBaseException { entityGraphMapper.updateTagPropagations(relationshipEdgeId, relationship); } } + + public static class CleanUpClassificationPropagation extends ClassificationTask { + public CleanUpClassificationPropagation(AtlasTask task, AtlasGraph graph, EntityGraphMapper entityGraphMapper, DeleteHandlerDelegate deleteDelegate, AtlasRelationshipStore relationshipStore) { + super(task, graph, entityGraphMapper, deleteDelegate, relationshipStore); + } + + @Override + protected void run(Map parameters) throws AtlasBaseException, IOException { + String classificationName = (String) parameters.get(PARAM_CLASSIFICATION_NAME); + + entityGraphMapper.cleanUpClassificationPropagation(classificationName); + } + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationTask.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationTask.java index 76112dd6852..71b56c1a517 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationTask.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationTask.java @@ -34,6 +34,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -50,6 +51,8 @@ public abstract class ClassificationTask extends AbstractTask { public static final String PARAM_RELATIONSHIP_GUID = "relationshipGuid"; public static final String PARAM_RELATIONSHIP_OBJECT = "relationshipObject"; public static final String PARAM_RELATIONSHIP_EDGE_ID = "relationshipEdgeId"; + + public static final String PARAM_CLASSIFICATION_NAME = "classificationName"; public static final String PARAM_REFERENCED_VERTEX_ID = "referencedVertexId"; public static final String PARAM_IS_TERM_ENTITY_EDGE = "isTermEntityEdge"; public static final String PARAM_PREVIOUS_CLASSIFICATION_RESTRICT_PROPAGATE_THROUGH_LINEAGE = "previousRestrictPropagationThroughLineage"; @@ -75,7 +78,7 @@ public ClassificationTask(AtlasTask task, } @Override - public AtlasTask.Status perform() throws AtlasBaseException { + public AtlasTask.Status perform() throws AtlasBaseException, IOException { Map params = getTaskDef().getParameters(); AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord(getTaskGuid()); @@ -101,7 +104,7 @@ public AtlasTask.Status perform() throws AtlasBaseException { run(params); setStatus(COMPLETE); - } catch (AtlasBaseException e) { + } catch (AtlasBaseException | IOException e) { LOG.error("Task: {}: Error performing task!", getTaskGuid(), e); setStatus(FAILED); @@ -177,5 +180,5 @@ protected void setStatus(AtlasTask.Status status) { graph.commit(); } - protected abstract void run(Map parameters) throws AtlasBaseException; + protected abstract void run(Map parameters) throws AtlasBaseException, IOException; } diff --git a/repository/src/main/java/org/apache/atlas/tasks/AtlasTaskService.java b/repository/src/main/java/org/apache/atlas/tasks/AtlasTaskService.java index ba0fe1a4da7..d8269633d43 100644 --- a/repository/src/main/java/org/apache/atlas/tasks/AtlasTaskService.java +++ b/repository/src/main/java/org/apache/atlas/tasks/AtlasTaskService.java @@ -148,7 +148,7 @@ public List createAtlasTasks(List tasks) throws AtlasBaseE if (!supportedTypes.contains(taskType)) { throw new AtlasBaseException(AtlasErrorCode.TASK_TYPE_NOT_SUPPORTED, task.getType()); } - if (isClassificationTaskType(taskType)) { + if (isClassificationTaskType(taskType) && !taskType.equals(ClassificationPropagateTaskFactory.CLEANUP_CLASSIFICATION_PROPAGATION)) { String classificationName = task.getClassificationName(); String entityGuid = task.getEntityGuid(); String classificationId = StringUtils.isEmpty(task.getClassificationId()) ? resolveAndReturnClassificationId(classificationName, entityGuid) : task.getClassificationId(); From eaf8f5b50cfa3bce2ac84ca4fa8a68fc03166dc8 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Tue, 21 May 2024 18:07:44 +0530 Subject: [PATCH 223/316] fix: add Autowired to the import --- .../atlas/repository/store/graph/v2/EntityGraphMapper.java | 1 + 1 file changed, 1 insertion(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 85286282c96..2710fb4b806 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -86,6 +86,7 @@ import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; From fd368cd9264386b417716e3d178655023cad9b24 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Tue, 21 May 2024 18:43:35 +0530 Subject: [PATCH 224/316] fix: add classificationVertex deletion step if remaining --- .../org/apache/atlas/repository/graph/GraphHelper.java | 7 +++++++ .../atlas/repository/store/graph/v2/EntityGraphMapper.java | 7 +++++++ 2 files changed, 14 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index d86c74414cf..de72caf72f7 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -376,6 +376,13 @@ public static AtlasVertex getClassificationVertex(AtlasVertex entityVertex, Stri return ret; } + public static List getAllClassificationVertex(AtlasGraph graph, String classificationName) { + Iterable vertices = graph.query().has(TYPE_NAME_PROPERTY_KEY, classificationName).vertices(); + if (vertices == null) { + return Collections.emptyList(); + } + return IteratorUtils.toList(vertices.iterator()); + } public static AtlasEdge getClassificationEdge(AtlasVertex entityVertex, AtlasVertex classificationVertex) { AtlasEdge ret = null; Iterable edges = entityVertex.query().direction(AtlasEdgeDirection.OUT).label(CLASSIFICATION_LABEL) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 2710fb4b806..c52353938e6 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -3028,6 +3028,13 @@ public void cleanUpClassificationPropagation(String classificationName) throws A transactionInterceptHelper.intercept(); LOG.info("Processed cleaning up {} entities", counter); } + + // Fetch all classificationVertex by classificationName and delete them if remaining + List classificationVertices = GraphHelper.getAllClassificationVertex(graph, classificationName); + for (AtlasVertex classificationVertex : classificationVertices) { + deleteDelegate.getHandler().deleteClassificationVertex(classificationVertex, true); + } + transactionInterceptHelper.intercept(); } } From f005278cb9ee79daa859a080ce26dfb19de9cef7 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Wed, 22 May 2024 09:37:34 +0530 Subject: [PATCH 225/316] fix: fix the edge case of no tag attached --- .../store/graph/v2/EntityGraphMapper.java | 37 ++++++++++--------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index c52353938e6..42f6456b36f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -3061,11 +3061,11 @@ public AtlasEntity repairClassificationMappings(AtlasVertex entityVertex) throws AtlasEntity entity = instanceConverter.getEntity(guid, ENTITY_CHANGE_NOTIFY_IGNORE_RELATIONSHIP_ATTRIBUTES); AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE_CLASSIFICATION, new AtlasEntityHeader(entity)), "repair classification mappings: guid=", guid); + List classificationNames = new ArrayList<>(); + List propagatedClassificationNames = new ArrayList<>(); if (entity.getClassifications() != null) { List classifications = entity.getClassifications(); - List classificationNames = new ArrayList<>(); - List propagatedClassificationNames = new ArrayList<>(); for (AtlasClassification classification : classifications) { if (isPropagatedClassification(classification, guid)) { propagatedClassificationNames.add(classification.getTypeName()); @@ -3073,25 +3073,26 @@ public AtlasEntity repairClassificationMappings(AtlasVertex entityVertex) throws classificationNames.add(classification.getTypeName()); } } - //Delete array/set properties first - entityVertex.removeProperty(TRAIT_NAMES_PROPERTY_KEY); - entityVertex.removeProperty(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY); + } + //Delete array/set properties first + entityVertex.removeProperty(TRAIT_NAMES_PROPERTY_KEY); + entityVertex.removeProperty(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY); - //Update classificationNames and propagatedClassificationNames in entityVertex - entityVertex.setProperty(CLASSIFICATION_NAMES_KEY, getDelimitedClassificationNames(classificationNames)); - entityVertex.setProperty(PROPAGATED_CLASSIFICATION_NAMES_KEY, getDelimitedClassificationNames(propagatedClassificationNames)); - entityVertex.setProperty(CLASSIFICATION_TEXT_KEY, fullTextMapperV2.getClassificationTextForEntity(entity)); - // Make classificationNames unique list as it is of type SET - classificationNames = classificationNames.stream().distinct().collect(Collectors.toList()); - //Update classificationNames and propagatedClassificationNames in entityHeader - for(String classificationName : classificationNames) { - AtlasGraphUtilsV2.addEncodedProperty(entityVertex, TRAIT_NAMES_PROPERTY_KEY, classificationName); - } - for (String classificationName : propagatedClassificationNames) { - entityVertex.addListProperty(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationName); - } + //Update classificationNames and propagatedClassificationNames in entityVertex + entityVertex.setProperty(CLASSIFICATION_NAMES_KEY, getDelimitedClassificationNames(classificationNames)); + entityVertex.setProperty(PROPAGATED_CLASSIFICATION_NAMES_KEY, getDelimitedClassificationNames(propagatedClassificationNames)); + entityVertex.setProperty(CLASSIFICATION_TEXT_KEY, fullTextMapperV2.getClassificationTextForEntity(entity)); + // Make classificationNames unique list as it is of type SET + classificationNames = classificationNames.stream().distinct().collect(Collectors.toList()); + //Update classificationNames and propagatedClassificationNames in entityHeader + for(String classificationName : classificationNames) { + AtlasGraphUtilsV2.addEncodedProperty(entityVertex, TRAIT_NAMES_PROPERTY_KEY, classificationName); + } + for (String classificationName : propagatedClassificationNames) { + entityVertex.addListProperty(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationName); } + return entity; } From 6505586f401f0ba767cc4e0fbdb07f8fccefe458 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Wed, 22 May 2024 15:22:39 +0530 Subject: [PATCH 226/316] Added error based on the PR comment --- .../v2/preprocessor/datamesh/DataProductPreProcessor.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 1927e79caf6..4c5f3d6a767 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -81,8 +81,7 @@ private void processCreateProduct(AtlasEntity entity,AtlasVertex vertex) throws String parentDomainQualifiedName = ""; if (parentDomainObject == null) { - entity.removeAttribute(PARENT_DOMAIN_QN_ATTR); - entity.removeAttribute(SUPER_DOMAIN_QN_ATTR); + throw new AtlasBaseException("Cannot create a Product without a Domain Relationship"); } else { AtlasVertex parentDomain = retrieverNoRelation.getEntityVertex(parentDomainObject); parentDomainQualifiedName = parentDomain.getProperty(QUALIFIED_NAME, String.class); From 1326de684a82e0b3dec40f975206688967cc6002 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Wed, 22 May 2024 21:21:14 +0530 Subject: [PATCH 227/316] Fixed the error code logic as per PR comment. --- .../v2/preprocessor/datamesh/DataProductPreProcessor.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index 4c5f3d6a767..ea974535b6c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -25,6 +25,7 @@ import java.util.*; +import static org.apache.atlas.AtlasErrorCode.OPERATION_NOT_SUPPORTED; import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.*; import static org.apache.atlas.repository.util.AccessControlUtils.*; @@ -81,7 +82,7 @@ private void processCreateProduct(AtlasEntity entity,AtlasVertex vertex) throws String parentDomainQualifiedName = ""; if (parentDomainObject == null) { - throw new AtlasBaseException("Cannot create a Product without a Domain Relationship"); + throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Cannot create a Product without a Domain Relationship"); } else { AtlasVertex parentDomain = retrieverNoRelation.getEntityVertex(parentDomainObject); parentDomainQualifiedName = parentDomain.getProperty(QUALIFIED_NAME, String.class); From 7af6d654193b1cea1d611ebc4f3e3a75f336281b Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Thu, 23 May 2024 18:07:04 +0530 Subject: [PATCH 228/316] feat: make janusgraph as SOT instead of ES while fetching assets --- .../repository/graphdb/AtlasIndexQuery.java | 2 - .../janus/AtlasElasticsearchQuery.java | 28 ----------- .../graphdb/janus/AtlasJanusIndexQuery.java | 6 --- .../discovery/AtlasDiscoveryService.java | 8 ---- .../discovery/EntityDiscoveryService.java | 45 +----------------- .../atlas/repository/graph/GraphHelper.java | 17 +++++++ .../store/graph/v2/EntityGraphMapper.java | 46 +++---------------- .../apache/atlas/tasks/TaskQueueWatcher.java | 9 ++-- 8 files changed, 29 insertions(+), 132 deletions(-) diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java index 0da85b796ca..525152b3099 100644 --- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java +++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java @@ -44,8 +44,6 @@ public interface AtlasIndexQuery { DirectIndexQueryResult vertices(SearchParams searchParams) throws AtlasBaseException ; - int getNumVertices(SearchParams searchParams) throws AtlasBaseException, IOException; - /** * Gets the query results form index * diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java index 2dddd1ef155..c3030291460 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java @@ -128,34 +128,6 @@ private Iterator> runQuery(SearchReques return result; } - public int getNumVertices(SearchParams searchParams) throws AtlasBaseException, IOException { - // Make a count query - String query = searchParams.getQuery(); - HttpEntity entity = new NStringEntity(query, ContentType.APPLICATION_JSON); - String endPoint = index + "/_count"; - Request request = new Request("GET", endPoint); - request.setEntity(entity); - Response response; - try { - response = lowLevelRestClient.performRequest(request); - } catch (ResponseException rex) { - if (rex.getResponse().getStatusLine().getStatusCode() == 404) { - LOG.warn(String.format("ES index with name %s not found", index)); - throw new AtlasBaseException(INDEX_NOT_FOUND, index); - } else { - throw new AtlasBaseException(rex); - } - } catch (IOException e) { - throw new AtlasBaseException(AtlasErrorCode.INDEX_SEARCH_FAILED, e.getMessage()); - } - return getVertexCountFromResponse(EntityUtils.toString(response.getEntity())); - - } - int getVertexCountFromResponse(String responseString) throws IOException { - Map responseMap = AtlasType.fromJson(responseString, Map.class); - return (Integer) responseMap.get("count"); - } - private DirectIndexQueryResult runQueryWithLowLevelClient(SearchParams searchParams) throws AtlasBaseException { DirectIndexQueryResult result = null; diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusIndexQuery.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusIndexQuery.java index ba680361d08..d4417e8f6e3 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusIndexQuery.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusIndexQuery.java @@ -17,7 +17,6 @@ */ package org.apache.atlas.repository.graphdb.janus; -import java.io.IOException; import java.util.*; import com.google.common.base.Preconditions; @@ -50,11 +49,6 @@ public DirectIndexQueryResult vertices(SearchP return null; } - @Override - public int getNumVertices(SearchParams searchParams) throws AtlasBaseException, IOException { - return 0; - } - @Override public Map directIndexQuery(String query) throws AtlasBaseException { return null; diff --git a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java index e898190d272..15a3a1f1996 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java @@ -159,14 +159,6 @@ AtlasSearchResult searchUsingBasicQuery(String query, String type, String classi */ AtlasSearchResult directIndexSearch(SearchParams searchParams) throws AtlasBaseException; - /** - * Search for direct ES query - * @param searchParams Search criteria - * @return Matching entities - * @throws AtlasBaseException - */ - DirectIndexQueryResult directIndexSearch(SearchParams searchParams, boolean returnVertices) throws AtlasBaseException; - /** * Search for direct ES query on search logs index * @param searchParams Search criteria diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index fb768cd6b89..66e8d589091 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -46,6 +46,7 @@ import org.apache.atlas.repository.userprofile.UserProfileService; import org.apache.atlas.repository.util.AccessControlUtils; import org.apache.atlas.searchlog.ESSearchLogger; +import org.apache.atlas.service.FeatureFlagStore; import org.apache.atlas.stats.StatsClient; import org.apache.atlas.type.*; import org.apache.atlas.type.AtlasBuiltInTypes.AtlasObjectIdType; @@ -69,7 +70,6 @@ import javax.inject.Inject; import javax.script.ScriptEngine; import javax.script.ScriptException; -import java.io.IOException; import java.util.*; import java.util.stream.Collectors; @@ -1013,49 +1013,6 @@ public AtlasSearchResult directIndexSearch(SearchParams searchParams) throws Atl return ret; } - public int getQueryResponseCount(SearchParams searchParams) throws AtlasBaseException, IOException { - IndexSearchParams params = (IndexSearchParams) searchParams; - String indexName = getIndexName(params); - AtlasIndexQuery indexQuery = graph.elasticsearchQuery(indexName); - return indexQuery.getNumVertices(searchParams); - - } - - @Override - public DirectIndexQueryResult directIndexSearch(SearchParams searchParams, boolean returnVertices) throws AtlasBaseException { - IndexSearchParams params = (IndexSearchParams) searchParams; - RequestContext.get().setRelationAttrsForSearch(params.getRelationAttributes()); - RequestContext.get().setAllowDeletedRelationsIndexsearch(params.isAllowDeletedRelations()); - - DirectIndexQueryResult ret = null; - AtlasIndexQuery indexQuery; - - Set resultAttributes = new HashSet<>(); - if (CollectionUtils.isNotEmpty(searchParams.getAttributes())) { - resultAttributes.addAll(searchParams.getAttributes()); - } - - try { - if(LOG.isDebugEnabled()){ - LOG.debug("Performing ES search for the params ({})", searchParams); - } - - String indexName = getIndexName(params); - - indexQuery = graph.elasticsearchQuery(indexName); - AtlasPerfMetrics.MetricRecorder elasticSearchQueryMetric = RequestContext.get().startMetricRecord("elasticSearchQuery"); - ret = indexQuery.vertices(searchParams); - if (ret == null) { - return null; - } - RequestContext.get().endMetricRecord(elasticSearchQueryMetric); - } catch (Exception e) { - LOG.error("Error while performing direct search for the params ({}), {}", searchParams, e.getMessage()); - throw e; - } - return ret; - } - @Override public SearchLogSearchResult searchLogs(SearchLogSearchParams searchParams) throws AtlasBaseException { SearchLogSearchResult ret = new SearchLogSearchResult(); diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index de72caf72f7..6b323d3ac00 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -383,6 +383,23 @@ public static List getAllClassificationVertex(AtlasGraph graph, St } return IteratorUtils.toList(vertices.iterator()); } + + public static List getAllAssetsWithClassificationAttached(AtlasGraph graph, String classificationName, int limit) { + AtlasGraphQuery query = graph.query(); + AtlasGraphQuery hasPropagatedTraitNames = query.createChildQuery().has(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationName); + AtlasGraphQuery hasTraitNames = query.createChildQuery().has(TRAIT_NAMES_PROPERTY_KEY, classificationName); + Iterable vertices = query.or( + Arrays.asList( + hasPropagatedTraitNames, + hasTraitNames + ) + ).vertices(); + if (vertices == null) { + return Collections.emptyList(); + } + + return IteratorUtils.toList(vertices.iterator()); + } public static AtlasEdge getClassificationEdge(AtlasVertex entityVertex, AtlasVertex classificationVertex) { AtlasEdge ret = null; Iterable edges = entityVertex.query().direction(AtlasEdgeDirection.OUT).label(CLASSIFICATION_LABEL) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 42f6456b36f..2f736bdf28a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -2977,34 +2977,18 @@ private void updateInConsistentOwnedMapVertices(AttributeMutationContext ctx, At } } - public void cleanUpClassificationPropagation(String classificationName) throws AtlasBaseException, IOException { - IndexSearchParams searchParams = new IndexSearchParams(); + public void cleanUpClassificationPropagation(String classificationName) throws AtlasBaseException { int batchSize = 100; - DirectIndexQueryResult indexQueryResult; - Map dsl = buildMap(classificationName); - searchParams.setDsl(dsl); - int totalCount = discoveryService.getQueryResponseCount(searchParams); - LOG.info("Total entities found for classification {} are {}", classificationName, totalCount); int counter = 0; - dsl.put("from", 0); - dsl.put("size", batchSize); - searchParams.setDsl(dsl); - while (counter <= totalCount) { + while (true) { try { - indexQueryResult = discoveryService.directIndexSearch(searchParams, true); - if (indexQueryResult == null) { - if (LOG.isDebugEnabled()) { - LOG.debug("No entities found for classification {}", classificationName); - } - return; - } - Iterator results = indexQueryResult.getIterator(); - if (!results.hasNext()) { + + List vertices = GraphHelper.getAllAssetsWithClassificationAttached(graph, classificationName, batchSize); + if (CollectionUtils.isEmpty(vertices)) { LOG.info("No entities found for classification {}", classificationName); return; } - while (results.hasNext()) { - AtlasVertex vertex = results.next().getVertex(); + for(AtlasVertex vertex : vertices) { String guid = GraphHelper.getGuid(vertex); GraphTransactionInterceptor.lockObjectAndReleasePostCommit(guid); List deletedClassifications = new ArrayList<>(); @@ -3038,24 +3022,6 @@ public void cleanUpClassificationPropagation(String classificationName) throws A } } - private Map getMap(String key, Object value) { - Map map = new HashMap<>(); - map.put(key, value); - return map; - } - - public Map buildMap(String classificationName) { - Map dsl = new HashMap<>(); - Map query = new HashMap<>(); - List shouldClauses = new ArrayList<>(); - Map terms1 = getMap("terms", getMap("__traitNames", Collections.singletonList(classificationName))); - Map terms2 = getMap("terms", getMap("__propagatedTraitNames", Collections.singletonList(classificationName))); - shouldClauses.add(terms1); - shouldClauses.add(terms2); - query.put("bool", getMap("should", shouldClauses)); - dsl.put("query", query); - return dsl; - } public AtlasEntity repairClassificationMappings(AtlasVertex entityVertex) throws AtlasBaseException { String guid = GraphHelper.getGuid(entityVertex); AtlasEntity entity = instanceConverter.getEntity(guid, ENTITY_CHANGE_NOTIFY_IGNORE_RELATIONSHIP_ATTRIBUTES); diff --git a/repository/src/main/java/org/apache/atlas/tasks/TaskQueueWatcher.java b/repository/src/main/java/org/apache/atlas/tasks/TaskQueueWatcher.java index 9214cbfbf9b..0c68d7df69f 100644 --- a/repository/src/main/java/org/apache/atlas/tasks/TaskQueueWatcher.java +++ b/repository/src/main/java/org/apache/atlas/tasks/TaskQueueWatcher.java @@ -88,10 +88,11 @@ public void run() { } while (shouldRun.get()) { try { - if (!redisService.acquireDistributedLock(ATLAS_TASK_LOCK)) { - Thread.sleep(AtlasConstants.TASK_WAIT_TIME_MS); - continue; - } +// if (!redisService.acquireDistributedLock(ATLAS_TASK_LOCK)) { +// Thread.sleep(AtlasConstants.TASK_WAIT_TIME_MS); +// continue; +// } + Thread.sleep(100); TasksFetcher fetcher = new TasksFetcher(registry); From 01d6dd5aaf9881add7c18f66c652f341246e3803 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Thu, 23 May 2024 18:09:05 +0530 Subject: [PATCH 229/316] nit: remove redudunt imports --- .../org/apache/atlas/repository/graphdb/AtlasIndexQuery.java | 1 - .../java/org/apache/atlas/discovery/AtlasDiscoveryService.java | 1 - 2 files changed, 2 deletions(-) diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java index 525152b3099..09fa0d6d060 100644 --- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java +++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java @@ -22,7 +22,6 @@ import org.apache.atlas.model.discovery.SearchParams; import org.apache.tinkerpop.gremlin.process.traversal.Order; -import java.io.IOException; import java.util.Iterator; import java.util.List; import java.util.Map; diff --git a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java index 15a3a1f1996..dc567be4fe3 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/AtlasDiscoveryService.java @@ -29,7 +29,6 @@ import org.apache.atlas.model.profile.AtlasUserSavedSearch; import org.apache.atlas.model.searchlog.SearchLogSearchParams; import org.apache.atlas.model.searchlog.SearchLogSearchResult; -import org.apache.atlas.repository.graphdb.DirectIndexQueryResult; import java.util.List; From 8c6b1dc314ae997e59c136629f41c4d6c220e82f Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Thu, 23 May 2024 18:11:02 +0530 Subject: [PATCH 230/316] fix: fix unwanted commented out code --- .../java/org/apache/atlas/tasks/TaskQueueWatcher.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/tasks/TaskQueueWatcher.java b/repository/src/main/java/org/apache/atlas/tasks/TaskQueueWatcher.java index 0c68d7df69f..9214cbfbf9b 100644 --- a/repository/src/main/java/org/apache/atlas/tasks/TaskQueueWatcher.java +++ b/repository/src/main/java/org/apache/atlas/tasks/TaskQueueWatcher.java @@ -88,11 +88,10 @@ public void run() { } while (shouldRun.get()) { try { -// if (!redisService.acquireDistributedLock(ATLAS_TASK_LOCK)) { -// Thread.sleep(AtlasConstants.TASK_WAIT_TIME_MS); -// continue; -// } - Thread.sleep(100); + if (!redisService.acquireDistributedLock(ATLAS_TASK_LOCK)) { + Thread.sleep(AtlasConstants.TASK_WAIT_TIME_MS); + continue; + } TasksFetcher fetcher = new TasksFetcher(registry); From 030a99eedb0ad2032148e997936fc479effbe1ad Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Thu, 23 May 2024 18:12:35 +0530 Subject: [PATCH 231/316] nit: remove unwanted exception and imports --- .../atlas/repository/store/graph/v2/EntityGraphMapper.java | 1 - .../store/graph/v2/tasks/ClassificationPropagationTasks.java | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 2f736bdf28a..b1680b9f31b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -90,7 +90,6 @@ import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; -import java.io.IOException; import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationPropagationTasks.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationPropagationTasks.java index a3a3e98633b..d1191d3aac4 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationPropagationTasks.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationPropagationTasks.java @@ -26,7 +26,6 @@ import org.apache.atlas.repository.store.graph.v2.EntityGraphMapper; import org.apache.atlas.type.AtlasType; -import java.io.IOException; import java.util.Map; import java.util.Set; @@ -129,7 +128,7 @@ public CleanUpClassificationPropagation(AtlasTask task, AtlasGraph graph, Entity } @Override - protected void run(Map parameters) throws AtlasBaseException, IOException { + protected void run(Map parameters) throws AtlasBaseException { String classificationName = (String) parameters.get(PARAM_CLASSIFICATION_NAME); entityGraphMapper.cleanUpClassificationPropagation(classificationName); From 3a5b34053873511cad519e090799c1ff11994df8 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Thu, 23 May 2024 18:15:29 +0530 Subject: [PATCH 232/316] nit: add a log to specify end of operation --- .../atlas/repository/store/graph/v2/EntityGraphMapper.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index b1680b9f31b..5f47df3d1e4 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -3018,6 +3018,8 @@ public void cleanUpClassificationPropagation(String classificationName) throws A deleteDelegate.getHandler().deleteClassificationVertex(classificationVertex, true); } transactionInterceptHelper.intercept(); + LOG.info("Completed cleaning up classification {}", classificationName); + } } From 0101cf0de6462a4f020d900d47d4e3869bf8162b Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Thu, 23 May 2024 18:25:22 +0530 Subject: [PATCH 233/316] fix: add limit to query --- .../java/org/apache/atlas/repository/graph/GraphHelper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index 6b323d3ac00..b277a7ffd0f 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -393,7 +393,7 @@ public static List getAllAssetsWithClassificationAttached(AtlasGrap hasPropagatedTraitNames, hasTraitNames ) - ).vertices(); + ).vertices(limit); if (vertices == null) { return Collections.emptyList(); } From 93bce7096039fe815bf94bd871a2a1a8c69c2aef Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Thu, 23 May 2024 18:32:24 +0530 Subject: [PATCH 234/316] fix: instead of return enter break --- .../atlas/repository/store/graph/v2/EntityGraphMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 5f47df3d1e4..ed02b31e3fa 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -2985,7 +2985,7 @@ public void cleanUpClassificationPropagation(String classificationName) throws A List vertices = GraphHelper.getAllAssetsWithClassificationAttached(graph, classificationName, batchSize); if (CollectionUtils.isEmpty(vertices)) { LOG.info("No entities found for classification {}", classificationName); - return; + break; } for(AtlasVertex vertex : vertices) { String guid = GraphHelper.getGuid(vertex); From 599e6aedd6e41f7ae3c672224e21591509515ff0 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Thu, 23 May 2024 18:36:58 +0530 Subject: [PATCH 235/316] nit: good indentation practice --- .../atlas/repository/converters/AtlasInstanceConverter.java | 2 +- .../java/org/apache/atlas/repository/graph/GraphHelper.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java index 5c5c0811392..14fd5563df9 100644 --- a/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java +++ b/repository/src/main/java/org/apache/atlas/repository/converters/AtlasInstanceConverter.java @@ -322,7 +322,7 @@ public AtlasEntity getAndCacheEntity(String guid, boolean ignoreRelationshipAttr return entity; } - public AtlasEntity getEntity(String guid,boolean ignoreRelationshipAttributes) throws AtlasBaseException { + public AtlasEntity getEntity(String guid, boolean ignoreRelationshipAttributes) throws AtlasBaseException { AtlasEntity entity = null; if (ignoreRelationshipAttributes) { entity = entityGraphRetrieverIgnoreRelationshipAttrs.toAtlasEntity(guid); diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index b277a7ffd0f..55ae9bf9c3b 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -376,7 +376,7 @@ public static AtlasVertex getClassificationVertex(AtlasVertex entityVertex, Stri return ret; } - public static List getAllClassificationVertex(AtlasGraph graph, String classificationName) { + public static List getAllClassificationVertex(AtlasGraph graph, String classificationName) { Iterable vertices = graph.query().has(TYPE_NAME_PROPERTY_KEY, classificationName).vertices(); if (vertices == null) { return Collections.emptyList(); @@ -384,7 +384,7 @@ public static List getAllClassificationVertex(AtlasGraph graph, St return IteratorUtils.toList(vertices.iterator()); } - public static List getAllAssetsWithClassificationAttached(AtlasGraph graph, String classificationName, int limit) { + public static List getAllAssetsWithClassificationAttached(AtlasGraph graph, String classificationName, int limit) { AtlasGraphQuery query = graph.query(); AtlasGraphQuery hasPropagatedTraitNames = query.createChildQuery().has(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationName); AtlasGraphQuery hasTraitNames = query.createChildQuery().has(TRAIT_NAMES_PROPERTY_KEY, classificationName); From 901fcbc0f70a4560c0853f3761bc9c9638e1ae74 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Thu, 23 May 2024 18:38:41 +0530 Subject: [PATCH 236/316] nit: remove ioexception as not required --- .../store/graph/v2/tasks/ClassificationTask.java | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationTask.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationTask.java index 71b56c1a517..f1796ad5bd6 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationTask.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/tasks/ClassificationTask.java @@ -34,7 +34,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; import java.util.HashMap; import java.util.Map; @@ -78,7 +77,7 @@ public ClassificationTask(AtlasTask task, } @Override - public AtlasTask.Status perform() throws AtlasBaseException, IOException { + public AtlasTask.Status perform() throws AtlasBaseException { Map params = getTaskDef().getParameters(); AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord(getTaskGuid()); @@ -104,7 +103,7 @@ public AtlasTask.Status perform() throws AtlasBaseException, IOException { run(params); setStatus(COMPLETE); - } catch (AtlasBaseException | IOException e) { + } catch (AtlasBaseException e) { LOG.error("Task: {}: Error performing task!", getTaskGuid(), e); setStatus(FAILED); @@ -180,5 +179,5 @@ protected void setStatus(AtlasTask.Status status) { graph.commit(); } - protected abstract void run(Map parameters) throws AtlasBaseException, IOException; + protected abstract void run(Map parameters) throws AtlasBaseException; } From 65a760c419bdb174f706195daf68e6476dce8643 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Fri, 24 May 2024 16:49:47 +0530 Subject: [PATCH 237/316] nit: address review comments --- .../atlas/repository/graph/GraphHelper.java | 2 +- .../store/graph/v2/EntityGraphMapper.java | 35 ++++++++++--------- .../graph/v2/bulkimport/MigrationImport.java | 2 +- 3 files changed, 21 insertions(+), 18 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index 55ae9bf9c3b..3eaba385cd6 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -376,7 +376,7 @@ public static AtlasVertex getClassificationVertex(AtlasVertex entityVertex, Stri return ret; } - public static List getAllClassificationVertex(AtlasGraph graph, String classificationName) { + public static List getAllClassificationVerticesByClassificationName(AtlasGraph graph, String classificationName) { Iterable vertices = graph.query().has(TYPE_NAME_PROPERTY_KEY, classificationName).vertices(); if (vertices == null) { return Collections.emptyList(); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index ed02b31e3fa..d5cec6de1de 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -24,12 +24,10 @@ import org.apache.atlas.authorize.AtlasAuthorizationUtils; import org.apache.atlas.authorize.AtlasEntityAccessRequest; import org.apache.atlas.authorize.AtlasPrivilege; -import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.exception.EntityNotFoundException; import org.apache.atlas.model.TimeBoundary; import org.apache.atlas.model.TypeCategory; -import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.instance.AtlasClassification; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntityHeader; @@ -55,8 +53,6 @@ import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; -import org.apache.atlas.repository.graphdb.DirectIndexQueryResult; -import org.apache.atlas.repository.graphdb.AtlasIndexQuery; import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; import org.apache.atlas.repository.store.graph.EntityGraphDiscoveryContext; import org.apache.atlas.repository.store.graph.v1.DeleteHandlerDelegate; @@ -86,11 +82,22 @@ import org.apache.commons.lang3.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.annotation.Lazy; import org.springframework.stereotype.Component; -import java.util.*; +import javax.inject.Inject; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.NoSuchElementException; +import java.util.Objects; +import java.util.Set; +import java.util.UUID; +import java.util.Date; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -196,15 +203,13 @@ public class EntityGraphMapper { private final EntityGraphRetriever entityRetriever; private final IFullTextMapper fullTextMapperV2; private final TaskManagement taskManagement; - - private final EntityDiscoveryService discoveryService; private final TransactionInterceptHelper transactionInterceptHelper; - @Autowired + @Inject public EntityGraphMapper(DeleteHandlerDelegate deleteDelegate, RestoreHandlerV1 restoreHandlerV1, AtlasTypeRegistry typeRegistry, AtlasGraph graph, AtlasRelationshipStore relationshipStore, IAtlasEntityChangeNotifier entityChangeNotifier, AtlasInstanceConverter instanceConverter, IFullTextMapper fullTextMapperV2, - TaskManagement taskManagement, TransactionInterceptHelper transactionInterceptHelper,@Lazy EntityDiscoveryService discoveryService) { + TaskManagement taskManagement, TransactionInterceptHelper transactionInterceptHelper) { this.restoreHandlerV1 = restoreHandlerV1; this.graphHelper = new GraphHelper(graph); this.deleteDelegate = deleteDelegate; @@ -216,9 +221,7 @@ public EntityGraphMapper(DeleteHandlerDelegate deleteDelegate, RestoreHandlerV1 this.entityRetriever = new EntityGraphRetriever(graph, typeRegistry); this.fullTextMapperV2 = fullTextMapperV2; this.taskManagement = taskManagement; - this.transactionInterceptHelper = transactionInterceptHelper; - this.discoveryService = discoveryService; - } + this.transactionInterceptHelper = transactionInterceptHelper;} @VisibleForTesting public void setTasksUseFlag(boolean value) { @@ -3006,14 +3009,14 @@ public void cleanUpClassificationPropagation(String classificationName) throws A counter++; } } catch (AtlasBaseException e) { - throw new AtlasBaseException(AtlasErrorCode.INDEX_SEARCH_FAILED, e); + throw e; } finally { transactionInterceptHelper.intercept(); LOG.info("Processed cleaning up {} entities", counter); } // Fetch all classificationVertex by classificationName and delete them if remaining - List classificationVertices = GraphHelper.getAllClassificationVertex(graph, classificationName); + List classificationVertices = GraphHelper.getAllClassificationVerticesByClassificationName(graph, classificationName); for (AtlasVertex classificationVertex : classificationVertices) { deleteDelegate.getHandler().deleteClassificationVertex(classificationVertex, true); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/MigrationImport.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/MigrationImport.java index df13282b614..0cc7c4a318b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/MigrationImport.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/MigrationImport.java @@ -126,7 +126,7 @@ private AtlasEntityStoreV2 createEntityStore(AtlasGraph graph, AtlasTypeRegistry AtlasRelationshipStore relationshipStore = new AtlasRelationshipStoreV2(graph, typeRegistry, deleteDelegate, entityChangeNotifier); EntityGraphMapper entityGraphMapper = new EntityGraphMapper(deleteDelegate, restoreHandlerV1, typeRegistry, - graph, relationshipStore, entityChangeNotifier, getInstanceConverter(graph), fullTextMapperV2, null, null, null); + graph, relationshipStore, entityChangeNotifier, getInstanceConverter(graph), fullTextMapperV2, null, null); AtlasRelationshipStoreV2 atlasRelationshipStoreV2 = new AtlasRelationshipStoreV2(graph, typeRegistry, deleteDelegate, entityChangeNotifier); return new AtlasEntityStoreV2(graph, deleteDelegate, restoreHandlerV1, typeRegistry, entityChangeNotifier, entityGraphMapper, null, atlasRelationshipStoreV2, null); From 1165616b8f38ead44daa6d808d4f0740b4d8904c Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Mon, 27 May 2024 19:26:08 +0530 Subject: [PATCH 238/316] DG-1297 Fix NPE after graph commit --- .../graph/v2/DataMeshQNMigrationService.java | 102 ++++++++++-------- .../store/graph/v2/MigrationService.java | 4 +- .../v2/preprocessor/PreProcessorUtils.java | 12 ++- .../apache/atlas/web/rest/MigrationREST.java | 46 +++++--- 4 files changed, 96 insertions(+), 68 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java index 2a6391fd880..9ffe0b8937b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java @@ -1,6 +1,5 @@ package org.apache.atlas.repository.store.graph.v2; -import jline.internal.Log; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.RequestContext; import org.apache.atlas.discovery.EntityDiscoveryService; @@ -21,9 +20,7 @@ import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Component; -import javax.inject.Inject; import java.util.*; import static org.apache.atlas.repository.Constants.*; @@ -33,7 +30,7 @@ import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_CATEGORY; import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_RESOURCES; -public class DataMeshQNMigrationService implements MigrationService, Runnable { +public class DataMeshQNMigrationService implements MigrationService { private static final Logger LOG = LoggerFactory.getLogger(DataMeshQNMigrationService.class); @@ -41,7 +38,8 @@ public class DataMeshQNMigrationService implements MigrationService, Runnable { private final EntityDiscoveryService discovery; private final EntityGraphRetriever entityRetriever; - protected final AtlasTypeRegistry typeRegistry; + private final AtlasTypeRegistry typeRegistry; + private final RedisService redisService; private Map updatedPolicyResources; private final int BATCH_SIZE = 20; @@ -54,29 +52,38 @@ public class DataMeshQNMigrationService implements MigrationService, Runnable { private boolean forceRegen; private final TransactionInterceptHelper transactionInterceptHelper; - public DataMeshQNMigrationService(AtlasEntityStore entityStore, EntityDiscoveryService discovery, EntityGraphRetriever entityRetriever, AtlasTypeRegistry typeRegistry, TransactionInterceptHelper transactionInterceptHelper, boolean forceRegen) { + public DataMeshQNMigrationService(AtlasEntityStore entityStore, EntityDiscoveryService discovery, EntityGraphRetriever entityRetriever, AtlasTypeRegistry typeRegistry, TransactionInterceptHelper transactionInterceptHelper, RedisService redisService, boolean forceRegen) { this.entityRetriever = entityRetriever; this.entityStore = entityStore; this.discovery = discovery; this.typeRegistry = typeRegistry; - this.updatedPolicyResources = new HashMap<>(); - this.counter = 0; + this.redisService = redisService; this.transactionInterceptHelper = transactionInterceptHelper; this.forceRegen = forceRegen; + + this.updatedPolicyResources = new HashMap<>(); + this.counter = 0; } - public Boolean startMigration() throws Exception{ + public void startMigration() throws Exception { + try { + redisService.putValue(DATA_MESH_QN, MigrationStatus.IN_PROGRESS.name()); - Set attributes = new HashSet<>(Arrays.asList(SUPER_DOMAIN_QN_ATTR, PARENT_DOMAIN_QN_ATTR, "__customAttributes")); + Set attributes = new HashSet<>(Arrays.asList(SUPER_DOMAIN_QN_ATTR, PARENT_DOMAIN_QN_ATTR, "__customAttributes")); - List entities = getEntity(DATA_DOMAIN_ENTITY_TYPE, attributes, null); + List entities = getEntity(DATA_DOMAIN_ENTITY_TYPE, attributes, null); - for (AtlasEntityHeader superDomain: entities) { - skipSuperDomain = false; - updateChunk(superDomain); + for (AtlasEntityHeader superDomain: entities) { + skipSuperDomain = false; + updateChunk(superDomain); + } + } catch (Exception e) { + LOG.error("Migration failed", e); + redisService.putValue(DATA_MESH_QN, MigrationStatus.FAILED.name()); + throw e; } - return Boolean.TRUE; + redisService.putValue(DATA_MESH_QN, MigrationStatus.SUCCESSFUL.name()); } private void updateChunk(AtlasEntityHeader atlasEntity) throws AtlasBaseException { @@ -86,8 +93,9 @@ private void updateChunk(AtlasEntityHeader atlasEntity) throws AtlasBaseExceptio try{ migrateDomainAttributes(vertex, "", ""); - commitChanges(); - LOG.info("Migrated qualified name for entity: {}", qualifiedName); + if (counter > 0) { + commitChanges(); + } } catch (AtlasBaseException e){ this.errorOccured = true; @@ -100,18 +108,14 @@ private void migrateDomainAttributes(AtlasVertex vertex, String parentDomainQual return; } - counter++; - String currentQualifiedName = vertex.getProperty(QUALIFIED_NAME,String.class); String updatedQualifiedName = createDomainQualifiedName(parentDomainQualifiedName); - LOG.info("Migrating qualified name for Domain: {}", currentQualifiedName); - Map updatedAttributes = new HashMap<>(); Map customAttributes = GraphHelper.getCustomAttributes(vertex); if(!this.forceRegen && customAttributes != null && customAttributes.get(MIGRATION_CUSTOM_ATTRIBUTE) != null && customAttributes.get(MIGRATION_CUSTOM_ATTRIBUTE).equals("true")){ - LOG.info("Entity already migrated for entity: {}", currentQualifiedName); + LOG.info("Entity already migrated: {}", currentQualifiedName); updatedQualifiedName = vertex.getProperty(QUALIFIED_NAME,String.class); @@ -120,40 +124,42 @@ private void migrateDomainAttributes(AtlasVertex vertex, String parentDomainQual } } else { + counter++; + LOG.info("Migrating qualified name for Domain: {} to {}", currentQualifiedName, updatedQualifiedName); superDomainQualifiedName = commitChangesInMemory(currentQualifiedName, updatedQualifiedName, parentDomainQualifiedName, superDomainQualifiedName, vertex, updatedAttributes); } if (!skipSuperDomain) { Iterator products = getAllChildrenVertices(vertex, DATA_PRODUCT_EDGE_LABEL); - - while (products.hasNext()) { - try { - AtlasVertex productVertex = products.next(); - if (Objects.nonNull(productVertex)) { - migrateDataProductAttributes(productVertex, updatedQualifiedName, superDomainQualifiedName); - } - if (skipSuperDomain) - break; + List productsList = new ArrayList<>(); + products.forEachRemaining(productsList::add); + + for (AtlasVertex productVertex : productsList) { + if (Objects.nonNull(productVertex)) { + migrateDataProductAttributes(productVertex, updatedQualifiedName, superDomainQualifiedName); + } else { + LOG.warn("Found null product vertex"); } - catch (NullPointerException e){ - LOG.error("Null Pointer Exception occured for subdomains with parent : " + currentQualifiedName, e); + + if (skipSuperDomain) { + break; } } // Get all children domains of current domain Iterator childDomains = getAllChildrenVertices(vertex, DOMAIN_PARENT_EDGE_LABEL); - - while (childDomains.hasNext()) { - try { - AtlasVertex childVertex = childDomains.next(); - if (Objects.nonNull(childVertex)) { - migrateDomainAttributes(childVertex, updatedQualifiedName, superDomainQualifiedName); - } - if (skipSuperDomain) - break; + List childDomainsList = new ArrayList<>(); + childDomains.forEachRemaining(childDomainsList::add); + + for (AtlasVertex childVertex : childDomainsList) { + if (Objects.nonNull(childVertex)) { + migrateDomainAttributes(childVertex, updatedQualifiedName, superDomainQualifiedName); + } else { + LOG.warn("Found null sub-domain vertex"); } - catch (NullPointerException e){ - LOG.error("Null Pointer Exception occured for subdomains with parent : " + currentQualifiedName, e); + + if (skipSuperDomain) { + break; } } @@ -178,6 +184,7 @@ public void commitChanges() { try { transactionInterceptHelper.intercept(); + LOG.info("Committed a batch to the graph"); } catch (Exception e){ this.skipSuperDomain = true; this.errorOccured = true; @@ -225,18 +232,17 @@ private void migrateDataProductAttributes(AtlasVertex vertex, String parentDomai return; } - counter++; - String currentQualifiedName = vertex.getProperty(QUALIFIED_NAME,String.class); String updatedQualifiedName = createProductQualifiedName(parentDomainQualifiedName); - LOG.info("Migrating qualified name for Product: {}", currentQualifiedName); Map customAttributes = GraphHelper.getCustomAttributes(vertex); if(!this.forceRegen && customAttributes != null && customAttributes.get(MIGRATION_CUSTOM_ATTRIBUTE) != null && customAttributes.get(MIGRATION_CUSTOM_ATTRIBUTE).equals("true")) { LOG.info("Product already migrated: {}", currentQualifiedName); } else { + counter++; + LOG.info("Migrating qualified name for Product: {} to {}", currentQualifiedName, updatedQualifiedName); vertex.setProperty(QUALIFIED_NAME, updatedQualifiedName); //Store domainPolicies and resources to be updated @@ -445,7 +451,9 @@ public static Map mapOf(String key, Object value) { @Override public void run() { try { + LOG.info("Starting migration: {}", DATA_MESH_QN); startMigration(); + LOG.info("Finished migration: {}", DATA_MESH_QN); } catch (Exception e) { LOG.error("Error running migration : {}",e.toString()); throw new RuntimeException(e); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/MigrationService.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/MigrationService.java index 91d80c65540..9e93999cd96 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/MigrationService.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/MigrationService.java @@ -1,5 +1,5 @@ package org.apache.atlas.repository.store.graph.v2; -public interface MigrationService { - Boolean startMigration() throws Exception; +public interface MigrationService extends Runnable { + void startMigration() throws Exception; } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index 25122adaf5a..2680e482079 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -60,12 +60,14 @@ public class PreProcessorUtils { public static final String DAAP_VISIBILITY_GROUPS_ATTR = "daapVisibilityGroups"; //Migration Constants - public static final String MIGRATION = "MIGRATION:"; - public static final String DATA_MESH_QN = MIGRATION + "DATA_MESH_QN"; - public static final String IN_PROGRESS = "IN_PROGRESS"; - public static final String SUCCESSFUL = "SUCCESSFUL"; + public static final String MIGRATION_TYPE_PREFIX = "MIGRATION:"; + public static final String DATA_MESH_QN = MIGRATION_TYPE_PREFIX + "DATA_MESH_QN"; - public static final String FAILED = "FAILED"; + public enum MigrationStatus { + IN_PROGRESS, + SUCCESSFUL, + FAILED; + } //Query models constants public static final String PREFIX_QUERY_QN = "default/collection/"; diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java index 1e57ec2f5b8..929c2ae3090 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java @@ -12,6 +12,7 @@ import org.apache.atlas.repository.graphdb.*; import org.apache.atlas.repository.store.graph.AtlasEntityStore; import org.apache.atlas.repository.store.graph.v2.*; +import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils; import org.apache.atlas.repository.store.users.KeycloakStore; import org.apache.atlas.service.redis.RedisService; import org.apache.atlas.transformer.PreProcessorPoliciesTransformer; @@ -38,7 +39,7 @@ import static org.apache.atlas.auth.client.keycloak.AtlasKeycloakClient.getKeycloakClient; import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.DATA_MESH_QN; -import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.MIGRATION; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.MIGRATION_TYPE_PREFIX; @Path("migration") @Singleton @@ -57,7 +58,6 @@ public class MigrationREST { private final PreProcessorPoliciesTransformer transformer; private KeycloakStore keycloakStore; private AtlasGraph graph; - DataMeshQNMigrationService dataMeshQNMigrationService; private final EntityGraphRetriever entityRetriever; private final RedisService redisService; @@ -67,7 +67,8 @@ public class MigrationREST { private final TransactionInterceptHelper transactionInterceptHelper; @Inject - public MigrationREST(AtlasEntityStore entityStore, AtlasGraph graph, RedisService redisService,EntityDiscoveryService discovery, EntityGraphRetriever entityRetriever, AtlasTypeRegistry typeRegistry, TransactionInterceptHelper transactionInterceptHelper) { + public MigrationREST(AtlasEntityStore entityStore, AtlasGraph graph, RedisService redisService, EntityDiscoveryService discovery, + EntityGraphRetriever entityRetriever, AtlasTypeRegistry typeRegistry, TransactionInterceptHelper transactionInterceptHelper) { this.entityStore = entityStore; this.graph = graph; this.transformer = new PreProcessorPoliciesTransformer(); @@ -82,21 +83,31 @@ public MigrationREST(AtlasEntityStore entityStore, AtlasGraph graph, RedisServic @POST @Path("submit") @Timed - public Boolean submit (@QueryParam("migrationType") String migrationType,@QueryParam("forceMigration") boolean forceMigration) throws Exception { + public Boolean submit (@QueryParam("migrationType") String migrationType, @QueryParam("forceMigration") boolean forceMigration) throws Exception { AtlasPerfTracer perf = null; MigrationService migrationService; + try { if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "MigrationREST.submit()"); + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "MigrationREST.submit(" + migrationType + ")"); } - if( (MIGRATION + migrationType).equals(DATA_MESH_QN)){ - if(Objects.isNull(forceMigration) ){ - forceMigration = false; - } - dataMeshQNMigrationService = new DataMeshQNMigrationService(entityStore, discovery, entityRetriever, typeRegistry, transactionInterceptHelper, forceMigration); - dataMeshQNMigrationService.run(); + + migrationType = MIGRATION_TYPE_PREFIX + migrationType; + + isMigrationInProgress(migrationType); + + switch (migrationType) { + case DATA_MESH_QN: + migrationService = new DataMeshQNMigrationService(entityStore, discovery, entityRetriever, typeRegistry, transactionInterceptHelper, redisService, forceMigration); + break; + + default: + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Type of migration is not valid: " + migrationType); } + Thread migrationThread = new Thread(migrationService); + migrationThread.start(); + } catch (Exception e) { LOG.error("Error while submitting migration", e); return Boolean.FALSE; @@ -106,19 +117,26 @@ public Boolean submit (@QueryParam("migrationType") String migrationType,@QueryP return Boolean.TRUE; } + private void isMigrationInProgress(String migrationType) throws AtlasBaseException { + String status = redisService.getValue(migrationType); + if (PreProcessorUtils.MigrationStatus.IN_PROGRESS.name().equals(status)) { + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, + String.format("Migration for %s is already in progress", migrationType)); + } + } @GET @Path("status") @Timed - public String getMigrationStatus(@QueryParam("migrationType") String migrationType) throws Exception{ + public String getMigrationStatus(@QueryParam("migrationType") String migrationType) throws Exception { AtlasPerfTracer perf = null; try { if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "MigrationREST.getMigrationStatus()"); + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "MigrationREST.getMigrationStatus(" + migrationType + ")"); } - String value = redisService.getValue(MIGRATION + migrationType); + String value = redisService.getValue(MIGRATION_TYPE_PREFIX + migrationType); return Objects.nonNull(value) ? value : "No Migration Found with this key"; } catch (Exception e) { From a8fc383af22abd977333f51441ea59beec134204 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Tue, 28 May 2024 01:53:45 +0530 Subject: [PATCH 239/316] DG-1297 Fix NPE after graph commit --- .../store/graph/v2/DataMeshQNMigrationService.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java index 9ffe0b8937b..9efd5d5906d 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java @@ -25,6 +25,7 @@ import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.Constants.POLICY_ENTITY_TYPE; +import static org.apache.atlas.repository.graph.GraphHelper.getActiveChildrenVertices; import static org.apache.atlas.repository.graph.GraphHelper.getAllChildrenVertices; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.*; import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_CATEGORY; @@ -130,7 +131,7 @@ private void migrateDomainAttributes(AtlasVertex vertex, String parentDomainQual } if (!skipSuperDomain) { - Iterator products = getAllChildrenVertices(vertex, DATA_PRODUCT_EDGE_LABEL); + Iterator products = getActiveChildrenVertices(vertex, DATA_PRODUCT_EDGE_LABEL); List productsList = new ArrayList<>(); products.forEachRemaining(productsList::add); @@ -147,7 +148,7 @@ private void migrateDomainAttributes(AtlasVertex vertex, String parentDomainQual } // Get all children domains of current domain - Iterator childDomains = getAllChildrenVertices(vertex, DOMAIN_PARENT_EDGE_LABEL); + Iterator childDomains = getActiveChildrenVertices(vertex, DOMAIN_PARENT_EDGE_LABEL); List childDomainsList = new ArrayList<>(); childDomains.forEachRemaining(childDomainsList::add); From 614759d4a35f1a72f366eaf5934c823882dd6dd7 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Tue, 28 May 2024 01:58:19 +0530 Subject: [PATCH 240/316] DG-1297 Fix NPE after graph commit --- .../store/graph/v2/DataMeshQNMigrationService.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java index 9efd5d5906d..9ffe0b8937b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java @@ -25,7 +25,6 @@ import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.Constants.POLICY_ENTITY_TYPE; -import static org.apache.atlas.repository.graph.GraphHelper.getActiveChildrenVertices; import static org.apache.atlas.repository.graph.GraphHelper.getAllChildrenVertices; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.*; import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_CATEGORY; @@ -131,7 +130,7 @@ private void migrateDomainAttributes(AtlasVertex vertex, String parentDomainQual } if (!skipSuperDomain) { - Iterator products = getActiveChildrenVertices(vertex, DATA_PRODUCT_EDGE_LABEL); + Iterator products = getAllChildrenVertices(vertex, DATA_PRODUCT_EDGE_LABEL); List productsList = new ArrayList<>(); products.forEachRemaining(productsList::add); @@ -148,7 +147,7 @@ private void migrateDomainAttributes(AtlasVertex vertex, String parentDomainQual } // Get all children domains of current domain - Iterator childDomains = getActiveChildrenVertices(vertex, DOMAIN_PARENT_EDGE_LABEL); + Iterator childDomains = getAllChildrenVertices(vertex, DOMAIN_PARENT_EDGE_LABEL); List childDomainsList = new ArrayList<>(); childDomains.forEachRemaining(childDomainsList::add); From 5da96aa66afe6b288df7c555d035b2dd1a7ede2f Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Tue, 28 May 2024 02:12:23 +0530 Subject: [PATCH 241/316] DG-1297 Fix NPE after graph commit --- .../repository/store/graph/v2/DataMeshQNMigrationService.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java index 9ffe0b8937b..7341e0703f8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshQNMigrationService.java @@ -170,7 +170,7 @@ private void migrateDomainAttributes(AtlasVertex vertex, String parentDomainQual } } - public void commitChanges() { + public void commitChanges() throws AtlasBaseException { try { updatePolicy(this.updatedPolicyResources); } catch (AtlasBaseException e) { @@ -178,6 +178,7 @@ public void commitChanges() { this.skipSuperDomain = true; LOG.error("Failed to update set of policies: ", e); LOG.error("Failed policies: {}", AtlasType.toJson(this.updatedPolicyResources)); + throw e; } finally { this.updatedPolicyResources.clear(); } @@ -189,6 +190,7 @@ public void commitChanges() { this.skipSuperDomain = true; this.errorOccured = true; LOG.error("Failed to commit set of assets: ", e); + throw e; } finally { this.counter = 0; } From 05d48a3e4e2635ee75b5656566d84c66d0676830 Mon Sep 17 00:00:00 2001 From: arpit-at Date: Wed, 29 May 2024 12:39:03 +0530 Subject: [PATCH 242/316] DG-1374 Connection-creator to be added as admin only if adminlist is empty --- .github/workflows/maven.yml | 1 + .../store/graph/v2/preprocessor/ConnectionPreProcessor.java | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 0977cb36a25..bb5955c5489 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,6 +26,7 @@ on: - development - master - lineageondemand + - connectionadmins jobs: build: diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java index 02fb63bbc81..a19fa873c52 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java @@ -176,7 +176,7 @@ private void processUpdateConnection(EntityMutationContext context, if (connection.hasAttribute(ATTR_ADMIN_USERS)) { List newAdminUsers = (List) connection.getAttribute(ATTR_ADMIN_USERS); List currentAdminUsers = (List) existingConnEntity.getAttribute(ATTR_ADMIN_USERS); - if (StringUtils.isNotEmpty(creatorUser) && !newAdminUsers.contains(creatorUser)) { + if (StringUtils.isNotEmpty(creatorUser) && CollectionUtils.isNotEmpty(currentAdminUsers)) { newAdminUsers.add(creatorUser); } From 983d3c3fdf8a08bfce4746e182aff3ab74c62a34 Mon Sep 17 00:00:00 2001 From: arpit-at Date: Wed, 29 May 2024 15:47:08 +0530 Subject: [PATCH 243/316] DG-1374 Connection-creator to be added as admin only if adminlist is empty --- .../store/graph/v2/preprocessor/ConnectionPreProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java index a19fa873c52..273513129c7 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java @@ -176,7 +176,7 @@ private void processUpdateConnection(EntityMutationContext context, if (connection.hasAttribute(ATTR_ADMIN_USERS)) { List newAdminUsers = (List) connection.getAttribute(ATTR_ADMIN_USERS); List currentAdminUsers = (List) existingConnEntity.getAttribute(ATTR_ADMIN_USERS); - if (StringUtils.isNotEmpty(creatorUser) && CollectionUtils.isNotEmpty(currentAdminUsers)) { + if (StringUtils.isNotEmpty(creatorUser) && CollectionUtils.isEmpty(newAdminUsers)) { newAdminUsers.add(creatorUser); } From 8e1e53dd227a1db7ba45dd8ee15b609eed38329a Mon Sep 17 00:00:00 2001 From: arpit-at Date: Wed, 29 May 2024 16:06:17 +0530 Subject: [PATCH 244/316] DG-1374 Remove from build --- .github/workflows/maven.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index bb5955c5489..0977cb36a25 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,7 +26,6 @@ on: - development - master - lineageondemand - - connectionadmins jobs: build: From 61a4bcdc29d0de51373e47552790cc8a9eb228dc Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Thu, 30 May 2024 13:41:21 +0530 Subject: [PATCH 245/316] fix: change logic to fetch vertices --- .github/workflows/maven.yml | 2 +- .../atlas/repository/graph/GraphHelper.java | 31 ++++++++----- .../store/graph/v2/EntityGraphMapper.java | 44 +++++++++---------- 3 files changed, 41 insertions(+), 36 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 0977cb36a25..a12a2505a44 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -25,7 +25,7 @@ on: - beta - development - master - - lineageondemand + - tagcleanup jobs: build: diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index 3eaba385cd6..d131e175fc9 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -384,21 +384,28 @@ public static List getAllClassificationVerticesByClassificationName return IteratorUtils.toList(vertices.iterator()); } - public static List getAllAssetsWithClassificationAttached(AtlasGraph graph, String classificationName, int limit) { - AtlasGraphQuery query = graph.query(); - AtlasGraphQuery hasPropagatedTraitNames = query.createChildQuery().has(PROPAGATED_TRAIT_NAMES_PROPERTY_KEY, classificationName); - AtlasGraphQuery hasTraitNames = query.createChildQuery().has(TRAIT_NAMES_PROPERTY_KEY, classificationName); - Iterable vertices = query.or( - Arrays.asList( - hasPropagatedTraitNames, - hasTraitNames - ) - ).vertices(limit); - if (vertices == null) { + public static List getAllAssetsWithClassificationAttached(AtlasGraph graph, String classificationName) { + Iterable classificationVertices = graph.query().has(TYPE_NAME_PROPERTY_KEY, classificationName).vertices(); + if (classificationVertices == null) { return Collections.emptyList(); } + List classificationVerticesList = IteratorUtils.toList(classificationVertices.iterator()); + HashSet classificationVerticesSet = new HashSet<>(); + for (AtlasVertex classificationVertex : classificationVerticesList) { + Iterable attachedVertices = classificationVertex.query() + .direction(AtlasEdgeDirection.IN) + .label(CLASSIFICATION_LABEL).vertices(); + if (attachedVertices != null) { + Iterator attachedVerticesIterator = attachedVertices.iterator(); + while (attachedVerticesIterator.hasNext()) { + classificationVerticesSet.add(attachedVerticesIterator.next()); + } + } + return new ArrayList<>(classificationVerticesSet); + } + // - return IteratorUtils.toList(vertices.iterator()); + return IteratorUtils.toList(classificationVertices.iterator()); } public static AtlasEdge getClassificationEdge(AtlasVertex entityVertex, AtlasVertex classificationVertex) { AtlasEdge ret = null; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index d5cec6de1de..24b4f30db98 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -2980,22 +2980,23 @@ private void updateInConsistentOwnedMapVertices(AttributeMutationContext ctx, At } public void cleanUpClassificationPropagation(String classificationName) throws AtlasBaseException { - int batchSize = 100; - int counter = 0; - while (true) { + List vertices = GraphHelper.getAllAssetsWithClassificationAttached(graph, classificationName); + int batchSize = 2; + int totalVertexSize = vertices.size(); + LOG.info("Clean up tag {} from {} entities", classificationName, totalVertexSize); + int toIndex; + int offset = 0; + do { + toIndex = Math.min((offset + batchSize), totalVertexSize); + List entityVertices = vertices.subList(offset, toIndex); + List impactedGuids = entityVertices.stream().map(GraphHelper::getGuid).collect(Collectors.toList()); try { - - List vertices = GraphHelper.getAllAssetsWithClassificationAttached(graph, classificationName, batchSize); - if (CollectionUtils.isEmpty(vertices)) { - LOG.info("No entities found for classification {}", classificationName); - break; - } - for(AtlasVertex vertex : vertices) { + GraphTransactionInterceptor.lockObjectAndReleasePostCommit(impactedGuids); + for (AtlasVertex vertex : entityVertices) { String guid = GraphHelper.getGuid(vertex); GraphTransactionInterceptor.lockObjectAndReleasePostCommit(guid); List deletedClassifications = new ArrayList<>(); List classificationEdges = GraphHelper.getClassificationEdges(vertex, null, classificationName); - for (AtlasEdge edge : classificationEdges) { AtlasClassification classification = entityRetriever.toAtlasClassification(edge.getInVertex()); deletedClassifications.add(classification); @@ -3006,24 +3007,21 @@ public void cleanUpClassificationPropagation(String classificationName) throws A entityChangeNotifier.onClassificationDeletedFromEntity(entity, deletedClassifications); - counter++; + offset += batchSize; } - } catch (AtlasBaseException e) { - throw e; } finally { transactionInterceptHelper.intercept(); - LOG.info("Processed cleaning up {} entities", counter); - } - - // Fetch all classificationVertex by classificationName and delete them if remaining - List classificationVertices = GraphHelper.getAllClassificationVerticesByClassificationName(graph, classificationName); - for (AtlasVertex classificationVertex : classificationVertices) { - deleteDelegate.getHandler().deleteClassificationVertex(classificationVertex, true); + LOG.info("Cleaned up {} entities", offset); } - transactionInterceptHelper.intercept(); - LOG.info("Completed cleaning up classification {}", classificationName); + } while (offset < totalVertexSize); + // Fetch all classificationVertex by classificationName and delete them if remaining + List classificationVertices = GraphHelper.getAllClassificationVerticesByClassificationName(graph, classificationName); + for (AtlasVertex classificationVertex : classificationVertices) { + deleteDelegate.getHandler().deleteClassificationVertex(classificationVertex, true); } + transactionInterceptHelper.intercept(); + LOG.info("Completed cleaning up classification {}", classificationName); } public AtlasEntity repairClassificationMappings(AtlasVertex entityVertex) throws AtlasBaseException { From 0f7a9dd12a6a9b07d9dc08f4b8211be0c13f2cab Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Thu, 30 May 2024 13:50:53 +0530 Subject: [PATCH 246/316] fix: fix an error for offsetting --- .../org/apache/atlas/repository/graph/GraphHelper.java | 2 ++ .../atlas/repository/store/graph/v2/EntityGraphMapper.java | 7 ++----- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index d131e175fc9..a76c73837dd 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -390,6 +390,7 @@ public static List getAllAssetsWithClassificationAttached(AtlasGrap return Collections.emptyList(); } List classificationVerticesList = IteratorUtils.toList(classificationVertices.iterator()); + LOG.info("classificationVerticesList size: {}", classificationVerticesList.size(); HashSet classificationVerticesSet = new HashSet<>(); for (AtlasVertex classificationVertex : classificationVerticesList) { Iterable attachedVertices = classificationVertex.query() @@ -400,6 +401,7 @@ public static List getAllAssetsWithClassificationAttached(AtlasGrap while (attachedVerticesIterator.hasNext()) { classificationVerticesSet.add(attachedVerticesIterator.next()); } + LOG.info("classificationVerticesSet size: {}", classificationVerticesSet.size(); } return new ArrayList<>(classificationVerticesSet); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 24b4f30db98..eec38245c16 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -2983,7 +2983,7 @@ public void cleanUpClassificationPropagation(String classificationName) throws A List vertices = GraphHelper.getAllAssetsWithClassificationAttached(graph, classificationName); int batchSize = 2; int totalVertexSize = vertices.size(); - LOG.info("Clean up tag {} from {} entities", classificationName, totalVertexSize); + LOG.info("To clean up tag {} from {} entities", classificationName, totalVertexSize); int toIndex; int offset = 0; do { @@ -2993,8 +2993,6 @@ public void cleanUpClassificationPropagation(String classificationName) throws A try { GraphTransactionInterceptor.lockObjectAndReleasePostCommit(impactedGuids); for (AtlasVertex vertex : entityVertices) { - String guid = GraphHelper.getGuid(vertex); - GraphTransactionInterceptor.lockObjectAndReleasePostCommit(guid); List deletedClassifications = new ArrayList<>(); List classificationEdges = GraphHelper.getClassificationEdges(vertex, null, classificationName); for (AtlasEdge edge : classificationEdges) { @@ -3006,9 +3004,8 @@ public void cleanUpClassificationPropagation(String classificationName) throws A AtlasEntity entity = repairClassificationMappings(vertex); entityChangeNotifier.onClassificationDeletedFromEntity(entity, deletedClassifications); - - offset += batchSize; } + offset += batchSize; } finally { transactionInterceptHelper.intercept(); LOG.info("Cleaned up {} entities", offset); From 80f9ab53630d76712b34d3b586c6bdb71d37dfc7 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Thu, 30 May 2024 13:58:56 +0530 Subject: [PATCH 247/316] fix: fix typo --- .../java/org/apache/atlas/repository/graph/GraphHelper.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index a76c73837dd..c0ac175374a 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -390,7 +390,7 @@ public static List getAllAssetsWithClassificationAttached(AtlasGrap return Collections.emptyList(); } List classificationVerticesList = IteratorUtils.toList(classificationVertices.iterator()); - LOG.info("classificationVerticesList size: {}", classificationVerticesList.size(); + LOG.info("classificationVerticesList size: {}", classificationVerticesList.size()); HashSet classificationVerticesSet = new HashSet<>(); for (AtlasVertex classificationVertex : classificationVerticesList) { Iterable attachedVertices = classificationVertex.query() @@ -401,7 +401,7 @@ public static List getAllAssetsWithClassificationAttached(AtlasGrap while (attachedVerticesIterator.hasNext()) { classificationVerticesSet.add(attachedVerticesIterator.next()); } - LOG.info("classificationVerticesSet size: {}", classificationVerticesSet.size(); + LOG.info("classificationVerticesSet size: {}", classificationVerticesSet.size()); } return new ArrayList<>(classificationVerticesSet); } From 1e52fefeb1449d04681abbdba9eecfcc1bf8bf60 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Thu, 30 May 2024 14:44:39 +0530 Subject: [PATCH 248/316] fix: a small mistake in batching --- .../atlas/repository/store/graph/v2/EntityGraphMapper.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index eec38245c16..3e473733a30 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -2981,13 +2981,12 @@ private void updateInConsistentOwnedMapVertices(AttributeMutationContext ctx, At public void cleanUpClassificationPropagation(String classificationName) throws AtlasBaseException { List vertices = GraphHelper.getAllAssetsWithClassificationAttached(graph, classificationName); - int batchSize = 2; int totalVertexSize = vertices.size(); LOG.info("To clean up tag {} from {} entities", classificationName, totalVertexSize); int toIndex; int offset = 0; do { - toIndex = Math.min((offset + batchSize), totalVertexSize); + toIndex = Math.min((offset + CHUNK_SIZE), totalVertexSize); List entityVertices = vertices.subList(offset, toIndex); List impactedGuids = entityVertices.stream().map(GraphHelper::getGuid).collect(Collectors.toList()); try { @@ -3005,7 +3004,7 @@ public void cleanUpClassificationPropagation(String classificationName) throws A entityChangeNotifier.onClassificationDeletedFromEntity(entity, deletedClassifications); } - offset += batchSize; + offset += CHUNK_SIZE; } finally { transactionInterceptHelper.intercept(); LOG.info("Cleaned up {} entities", offset); From d7ca4daa09c5c8757ab1169b5b09c36cf67b40ea Mon Sep 17 00:00:00 2001 From: arpit-at Date: Thu, 30 May 2024 16:00:57 +0530 Subject: [PATCH 249/316] DG-1374 Update in pre-processor to handle the adminlist --- .github/workflows/maven.yml | 1 + .../src/main/java/org/apache/atlas/AtlasErrorCode.java | 4 ++-- .../graph/v2/preprocessor/ConnectionPreProcessor.java | 7 ++++--- .../preprocessor/sql/QueryCollectionPreProcessor.java | 10 ++++++---- 4 files changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 0977cb36a25..bb5955c5489 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,6 +26,7 @@ on: - development - master - lineageondemand + - connectionadmins jobs: build: diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java index fd2766616f7..457f00e562e 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java +++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java @@ -290,8 +290,8 @@ public enum AtlasErrorCode { TASK_INVALID_PARAMETERS(400, "ATLAS-400-00-111", "Invalid parameters for task {0}"), TASK_TYPE_NOT_SUPPORTED(400, "ATLAS-400-00-112", "Task type {0} is not supported"), - PERSONA_POLICY_ASSETS_LIMIT_EXCEEDED(400, "ATLAS-400-00-113", "Exceeded limit of maximum allowed assets across policies for a Persona: Limit: {0}, assets: {1}"); - + PERSONA_POLICY_ASSETS_LIMIT_EXCEEDED(400, "ATLAS-400-00-113", "Exceeded limit of maximum allowed assets across policies for a Persona: Limit: {0}, assets: {1}"), + ADMIN_LIST_SHOULD_NOT_BE_EMPTY(400, "ATLAS-400-00-113", "Admin list should not be empty for type {0}"); private String errorCode; private String errorMessage; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java index 273513129c7..1ffb7b26b62 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java @@ -17,6 +17,7 @@ */ package org.apache.atlas.repository.store.graph.v2.preprocessor; +import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.DeleteType; import org.apache.atlas.RequestContext; import org.apache.atlas.discovery.EntityDiscoveryService; @@ -171,13 +172,13 @@ private void processUpdateConnection(EntityMutationContext context, entity.setAttribute(QUALIFIED_NAME, vertexQName); RoleRepresentation representation = getKeycloakClient().getRoleByName(roleName); - String creatorUser = vertex.getProperty(CREATED_BY_KEY, String.class); + // String creatorUser = vertex.getProperty(CREATED_BY_KEY, String.class); if (connection.hasAttribute(ATTR_ADMIN_USERS)) { List newAdminUsers = (List) connection.getAttribute(ATTR_ADMIN_USERS); List currentAdminUsers = (List) existingConnEntity.getAttribute(ATTR_ADMIN_USERS); - if (StringUtils.isNotEmpty(creatorUser) && CollectionUtils.isEmpty(newAdminUsers)) { - newAdminUsers.add(creatorUser); + if (CollectionUtils.isEmpty(newAdminUsers)) { + throw new AtlasBaseException(AtlasErrorCode.ADMIN_LIST_SHOULD_NOT_BE_EMPTY, connection.getTypeName()); } connection.setAttribute(ATTR_ADMIN_USERS, newAdminUsers); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/sql/QueryCollectionPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/sql/QueryCollectionPreProcessor.java index 5dec4d31907..663b3e6423a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/sql/QueryCollectionPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/sql/QueryCollectionPreProcessor.java @@ -234,16 +234,18 @@ private void updateCollectionAdminRole(AtlasEntity collection, AtlasEntityHeader String adminRoleName = String.format(COLL_ADMIN_ROLE_PATTERN, collection.getGuid()); RoleRepresentation representation = getKeycloakClient().getRoleByName(adminRoleName); - String creatorUser = vertex.getProperty(CREATED_BY_KEY, String.class); + //String creatorUser = vertex.getProperty(CREATED_BY_KEY, String.class); if (collection.hasAttribute(ATTR_ADMIN_USERS)) { List newAdminUsers = (List) collection.getAttribute(ATTR_ADMIN_USERS); List currentAdminUsers = (List) existingCollEntity.getAttribute(ATTR_ADMIN_USERS); if (CollectionUtils.isNotEmpty(newAdminUsers) || CollectionUtils.isNotEmpty(currentAdminUsers)) { - if (StringUtils.isNotEmpty(creatorUser) && !newAdminUsers.contains(creatorUser)) { - newAdminUsers.add(creatorUser); - } + /* + if (StringUtils.isNotEmpty(creatorUser) && !newAdminUsers.contains(creatorUser)) { + newAdminUsers.add(creatorUser); + } + */ collection.setAttribute(ATTR_ADMIN_USERS, newAdminUsers); keycloakStore.updateRoleUsers(adminRoleName, currentAdminUsers, newAdminUsers, representation); } From 1e26506fc22854d9d2d6fe656400a88641121c6f Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Thu, 30 May 2024 16:09:24 +0530 Subject: [PATCH 250/316] fix: send set --- .../java/org/apache/atlas/repository/graph/GraphHelper.java | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index c0ac175374a..883bc6e01f6 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -403,11 +403,9 @@ public static List getAllAssetsWithClassificationAttached(AtlasGrap } LOG.info("classificationVerticesSet size: {}", classificationVerticesSet.size()); } - return new ArrayList<>(classificationVerticesSet); } - // - return IteratorUtils.toList(classificationVertices.iterator()); + return new ArrayList<>(classificationVerticesSet); } public static AtlasEdge getClassificationEdge(AtlasVertex entityVertex, AtlasVertex classificationVertex) { AtlasEdge ret = null; From 9812fe8ed5a2f2213dca65efe249344f3273f5fa Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Thu, 30 May 2024 17:01:17 +0530 Subject: [PATCH 251/316] Update GraphHelper.java --- .../org/apache/atlas/repository/graph/GraphHelper.java | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index 883bc6e01f6..9c687c00726 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -73,6 +73,7 @@ import java.util.Map; import java.util.Objects; import java.util.Set; +import java.util.stream.Collectors; import static org.apache.atlas.AtlasErrorCode.RELATIONSHIP_CREATE_INVALID_PARAMS; import static org.apache.atlas.model.instance.AtlasEntity.Status.ACTIVE; @@ -391,7 +392,7 @@ public static List getAllAssetsWithClassificationAttached(AtlasGrap } List classificationVerticesList = IteratorUtils.toList(classificationVertices.iterator()); LOG.info("classificationVerticesList size: {}", classificationVerticesList.size()); - HashSet classificationVerticesSet = new HashSet<>(); + HashSet entityVerticesSet = new HashSet<>(); for (AtlasVertex classificationVertex : classificationVerticesList) { Iterable attachedVertices = classificationVertex.query() .direction(AtlasEdgeDirection.IN) @@ -399,13 +400,13 @@ public static List getAllAssetsWithClassificationAttached(AtlasGrap if (attachedVertices != null) { Iterator attachedVerticesIterator = attachedVertices.iterator(); while (attachedVerticesIterator.hasNext()) { - classificationVerticesSet.add(attachedVerticesIterator.next()); + entityVerticesSet.add(attachedVerticesIterator.next()); } - LOG.info("classificationVerticesSet size: {}", classificationVerticesSet.size()); + LOG.info("classificationVerticesSet size: {}", entityVerticesSet.size()); } } - return new ArrayList<>(classificationVerticesSet); + return entityVerticesSet.stream().collect(Collectors.toList()); } public static AtlasEdge getClassificationEdge(AtlasVertex entityVertex, AtlasVertex classificationVertex) { AtlasEdge ret = null; From eac65117216bc4f80ad8de1c21dc29bfb538eece Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Thu, 30 May 2024 17:46:02 +0530 Subject: [PATCH 252/316] nit: add proper naming --- .github/workflows/maven.yml | 2 +- .../java/org/apache/atlas/repository/graph/GraphHelper.java | 4 ++-- .../atlas/repository/store/graph/v2/EntityGraphMapper.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index a12a2505a44..0977cb36a25 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -25,7 +25,7 @@ on: - beta - development - master - - tagcleanup + - lineageondemand jobs: build: diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index 9c687c00726..179d915df98 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -391,7 +391,7 @@ public static List getAllAssetsWithClassificationAttached(AtlasGrap return Collections.emptyList(); } List classificationVerticesList = IteratorUtils.toList(classificationVertices.iterator()); - LOG.info("classificationVerticesList size: {}", classificationVerticesList.size()); + LOG.info("classificationVerticesList size: {}", classificationVerticesList.size()); HashSet entityVerticesSet = new HashSet<>(); for (AtlasVertex classificationVertex : classificationVerticesList) { Iterable attachedVertices = classificationVertex.query() @@ -402,7 +402,7 @@ public static List getAllAssetsWithClassificationAttached(AtlasGrap while (attachedVerticesIterator.hasNext()) { entityVerticesSet.add(attachedVerticesIterator.next()); } - LOG.info("classificationVerticesSet size: {}", entityVerticesSet.size()); + LOG.info("entityVerticesSet size: {}", entityVerticesSet.size()); } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 3e473733a30..97ff2778726 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -3007,7 +3007,7 @@ public void cleanUpClassificationPropagation(String classificationName) throws A offset += CHUNK_SIZE; } finally { transactionInterceptHelper.intercept(); - LOG.info("Cleaned up {} entities", offset); + LOG.info("Cleaned up {} entities for classification {}", offset, classificationName); } } while (offset < totalVertexSize); From fde9b2b5f1b0315c49162e29256a57ca9015c5a4 Mon Sep 17 00:00:00 2001 From: vanshaj Date: Thu, 30 May 2024 18:52:25 +0530 Subject: [PATCH 253/316] Changed maximum lenght limit for attributes --- webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java index 0ebb48b16c6..10dc6e6f230 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java @@ -97,6 +97,8 @@ public class EntityREST { private static final int TWO_MILLION = HUNDRED_THOUSAND * 10 * 2; private static final Set ATTRS_WITH_TWO_MILLION_LIMIT = new HashSet() {{ add("rawQueryText"); + add("variablesSchemaBase64"); + add("visualBuilderSchemaBase64"); }}; From a21e0f7962e2730d9318b255e5a6a8da14416ea1 Mon Sep 17 00:00:00 2001 From: arpit-at Date: Fri, 31 May 2024 12:48:21 +0530 Subject: [PATCH 254/316] DG-1374 Update in pre-processor to handle the adminlist --- .github/workflows/maven.yml | 1 - .../graph/v2/preprocessor/ConnectionPreProcessor.java | 7 +++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index bb5955c5489..0977cb36a25 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,7 +26,6 @@ on: - development - master - lineageondemand - - connectionadmins jobs: build: diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java index 1ffb7b26b62..273513129c7 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java @@ -17,7 +17,6 @@ */ package org.apache.atlas.repository.store.graph.v2.preprocessor; -import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.DeleteType; import org.apache.atlas.RequestContext; import org.apache.atlas.discovery.EntityDiscoveryService; @@ -172,13 +171,13 @@ private void processUpdateConnection(EntityMutationContext context, entity.setAttribute(QUALIFIED_NAME, vertexQName); RoleRepresentation representation = getKeycloakClient().getRoleByName(roleName); - // String creatorUser = vertex.getProperty(CREATED_BY_KEY, String.class); + String creatorUser = vertex.getProperty(CREATED_BY_KEY, String.class); if (connection.hasAttribute(ATTR_ADMIN_USERS)) { List newAdminUsers = (List) connection.getAttribute(ATTR_ADMIN_USERS); List currentAdminUsers = (List) existingConnEntity.getAttribute(ATTR_ADMIN_USERS); - if (CollectionUtils.isEmpty(newAdminUsers)) { - throw new AtlasBaseException(AtlasErrorCode.ADMIN_LIST_SHOULD_NOT_BE_EMPTY, connection.getTypeName()); + if (StringUtils.isNotEmpty(creatorUser) && CollectionUtils.isEmpty(newAdminUsers)) { + newAdminUsers.add(creatorUser); } connection.setAttribute(ATTR_ADMIN_USERS, newAdminUsers); From 6f81b2b356e4c7a62bdba5df61c71a8b7f8d3ab8 Mon Sep 17 00:00:00 2001 From: arpit-at Date: Fri, 31 May 2024 12:50:17 +0530 Subject: [PATCH 255/316] DG-1374 Update in pre-processor to handle the adminlist --- .../graph/v2/preprocessor/ConnectionPreProcessor.java | 7 ++++--- .../preprocessor/sql/QueryCollectionPreProcessor.java | 10 ++++------ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java index 273513129c7..1ffb7b26b62 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java @@ -17,6 +17,7 @@ */ package org.apache.atlas.repository.store.graph.v2.preprocessor; +import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.DeleteType; import org.apache.atlas.RequestContext; import org.apache.atlas.discovery.EntityDiscoveryService; @@ -171,13 +172,13 @@ private void processUpdateConnection(EntityMutationContext context, entity.setAttribute(QUALIFIED_NAME, vertexQName); RoleRepresentation representation = getKeycloakClient().getRoleByName(roleName); - String creatorUser = vertex.getProperty(CREATED_BY_KEY, String.class); + // String creatorUser = vertex.getProperty(CREATED_BY_KEY, String.class); if (connection.hasAttribute(ATTR_ADMIN_USERS)) { List newAdminUsers = (List) connection.getAttribute(ATTR_ADMIN_USERS); List currentAdminUsers = (List) existingConnEntity.getAttribute(ATTR_ADMIN_USERS); - if (StringUtils.isNotEmpty(creatorUser) && CollectionUtils.isEmpty(newAdminUsers)) { - newAdminUsers.add(creatorUser); + if (CollectionUtils.isEmpty(newAdminUsers)) { + throw new AtlasBaseException(AtlasErrorCode.ADMIN_LIST_SHOULD_NOT_BE_EMPTY, connection.getTypeName()); } connection.setAttribute(ATTR_ADMIN_USERS, newAdminUsers); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/sql/QueryCollectionPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/sql/QueryCollectionPreProcessor.java index 663b3e6423a..5dec4d31907 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/sql/QueryCollectionPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/sql/QueryCollectionPreProcessor.java @@ -234,18 +234,16 @@ private void updateCollectionAdminRole(AtlasEntity collection, AtlasEntityHeader String adminRoleName = String.format(COLL_ADMIN_ROLE_PATTERN, collection.getGuid()); RoleRepresentation representation = getKeycloakClient().getRoleByName(adminRoleName); - //String creatorUser = vertex.getProperty(CREATED_BY_KEY, String.class); + String creatorUser = vertex.getProperty(CREATED_BY_KEY, String.class); if (collection.hasAttribute(ATTR_ADMIN_USERS)) { List newAdminUsers = (List) collection.getAttribute(ATTR_ADMIN_USERS); List currentAdminUsers = (List) existingCollEntity.getAttribute(ATTR_ADMIN_USERS); if (CollectionUtils.isNotEmpty(newAdminUsers) || CollectionUtils.isNotEmpty(currentAdminUsers)) { - /* - if (StringUtils.isNotEmpty(creatorUser) && !newAdminUsers.contains(creatorUser)) { - newAdminUsers.add(creatorUser); - } - */ + if (StringUtils.isNotEmpty(creatorUser) && !newAdminUsers.contains(creatorUser)) { + newAdminUsers.add(creatorUser); + } collection.setAttribute(ATTR_ADMIN_USERS, newAdminUsers); keycloakStore.updateRoleUsers(adminRoleName, currentAdminUsers, newAdminUsers, representation); } From dc43a3be4506e667bc160b3fc81248e5611ea25e Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Fri, 31 May 2024 17:36:38 +0800 Subject: [PATCH 256/316] Logic for internal attributes for products --- .../apache/atlas/repository/Constants.java | 3 +- .../store/graph/v2/EntityGraphMapper.java | 148 ++++++++++++++++++ .../v2/preprocessor/PreProcessorUtils.java | 2 + 3 files changed, 152 insertions(+), 1 deletion(-) diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 95a6e0c0078..a1a0e1278f9 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -149,7 +149,8 @@ public final class Constants { public static final String REL_DOMAIN_TO_STAKEHOLDERS = "data_domain_stakeholders"; public static final String REL_STAKEHOLDER_TITLE_TO_STAKEHOLDERS = "stakeholder_title_stakeholders"; - + public static final String INPUT_PORT_PRODUCT_EDGE_LABEL = "__Asset.inputPortDataProducts"; + public static final String OUTPUT_PORT_PRODUCT_EDGE_LABEL = "__Asset.outputPortDataProducts"; /** * SQL property keys. diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 97ff2778726..9b69d2e919d 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -137,6 +137,8 @@ import static org.apache.atlas.repository.graph.GraphHelper.getPropagatableClassifications; import static org.apache.atlas.repository.graph.GraphHelper.getClassificationEntityGuid; import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.*; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.INPUT_PORT_GUIDS_ATTR; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.OUTPUT_PORT_GUIDS_ATTR; import static org.apache.atlas.repository.store.graph.v2.tasks.ClassificationPropagateTaskFactory.*; import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.IN; import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.AtlasRelationshipEdgeDirection.OUT; @@ -169,6 +171,7 @@ public class EntityGraphMapper { private static final String TYPE_GLOSSARY= "AtlasGlossary"; private static final String TYPE_CATEGORY= "AtlasGlossaryCategory"; private static final String TYPE_TERM = "AtlasGlossaryTerm"; + private static final String TYPE_PRODUCT = "DataProduct"; private static final String TYPE_PROCESS = "Process"; private static final String ATTR_MEANINGS = "meanings"; private static final String ATTR_ANCHOR = "anchor"; @@ -2002,6 +2005,11 @@ public List mapArrayValue(AttributeMutationContext ctx, EntityMutationContext co case PROCESS_INPUTS: case PROCESS_OUTPUTS: addEdgesToContext(GraphHelper.getGuid(ctx.referringVertex), newElementsCreated, removedElements); break; + + case INPUT_PORT_PRODUCT_EDGE_LABEL: + case OUTPUT_PORT_PRODUCT_EDGE_LABEL: + addInternalProductAttrForAppendAndRemove(ctx, newElementsCreated, removedElements, currentElements); + break; } if (LOG.isDebugEnabled()) { @@ -2028,9 +2036,16 @@ public List appendArrayValue(AttributeMutationContext ctx, EntityMutationContext AtlasAttribute inverseRefAttribute = attribute.getInverseRefAttribute(); Cardinality cardinality = attribute.getAttributeDef().getCardinality(); List newElementsCreated = new ArrayList<>(); + List currentElements; boolean isNewElementsNull = newElements == null; + if (isReference && !isSoftReference) { + currentElements = (List) getCollectionElementsUsingRelationship(ctx.getReferringVertex(), attribute, isStructType); + } else { + currentElements = (List) getArrayElementsProperty(elementType, isSoftReference, ctx.getReferringVertex(), ctx.getVertexProperty()); + } + if (isNewElementsNull) { newElements = new ArrayList(); } @@ -2087,6 +2102,11 @@ public List appendArrayValue(AttributeMutationContext ctx, EntityMutationContext case PROCESS_INPUTS: case PROCESS_OUTPUTS: addEdgesToContext(GraphHelper.getGuid(ctx.referringVertex), newElementsCreated, new ArrayList<>(0)); break; + + case INPUT_PORT_PRODUCT_EDGE_LABEL: + case OUTPUT_PORT_PRODUCT_EDGE_LABEL: + addInternalProductAttrForAppendAndRemove(ctx, newElementsCreated, new ArrayList<>(0), currentElements); + break; } if (LOG.isDebugEnabled()) { @@ -2110,10 +2130,19 @@ public List removeArrayValue(AttributeMutationContext ctx, EntityMutationContext Cardinality cardinality = attribute.getAttributeDef().getCardinality(); List removedElements = new ArrayList<>(); List entityRelationsDeleted = new ArrayList<>(); + boolean isReference = isReference(elementType); + boolean isSoftReference = ctx.getAttribute().getAttributeDef().isSoftReferenced(); + List currentElements; boolean isNewElementsNull = elementsDeleted == null; + if (isReference && !isSoftReference) { + currentElements = (List) getCollectionElementsUsingRelationship(ctx.getReferringVertex(), attribute, isStructType); + } else { + currentElements = (List) getArrayElementsProperty(elementType, isSoftReference, ctx.getReferringVertex(), ctx.getVertexProperty()); + } + if (isNewElementsNull) { elementsDeleted = new ArrayList(); } @@ -2154,6 +2183,11 @@ public List removeArrayValue(AttributeMutationContext ctx, EntityMutationContext case PROCESS_INPUTS: case PROCESS_OUTPUTS: addEdgesToContext(GraphHelper.getGuid(ctx.referringVertex), new ArrayList<>(0), removedElements); break; + + case INPUT_PORT_PRODUCT_EDGE_LABEL: + case OUTPUT_PORT_PRODUCT_EDGE_LABEL: + addInternalProductAttrForAppendAndRemove(ctx, new ArrayList<>(0) , removedElements, currentElements); + break; } if (LOG.isDebugEnabled()) { @@ -2191,6 +2225,120 @@ private void addEdgesToContext(String guid, List newElementsCreated, Lis } } + private void addInternalProductAttr (AttributeMutationContext ctx, List allArrayElements, List deletedElements) { + MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("addInternalProductAttr"); + AtlasVertex toVertex = ctx.getReferringVertex(); + String toVertexType = getTypeName(toVertex); + + if (TYPE_PRODUCT.equals(toVertexType)) { + List allArrayElementGuids = allArrayElements.stream() + .map(x -> ((AtlasEdge) x).getOutVertex().getProperty("__guid", String.class)) + .collect(Collectors.toList()); + + // Removing deleted elements from allArrayElementGuids + if (CollectionUtils.isNotEmpty(deletedElements)) { + List deletedGuids = deletedElements.stream() + .map(x -> x.getOutVertex().getProperty("__guid", String.class)) + .collect(Collectors.toList()); + allArrayElementGuids.removeAll(deletedGuids); + } + + if(ctx.getAttribute().getRelationshipEdgeLabel().equals(OUTPUT_PORT_PRODUCT_EDGE_LABEL)){ + List outputPortGuids = null; + outputPortGuids = allArrayElementGuids; + toVertex.removeProperty(OUTPUT_PORT_GUIDS_ATTR); + if (CollectionUtils.isNotEmpty(outputPortGuids)) { + outputPortGuids.forEach(guid -> AtlasGraphUtilsV2.addEncodedProperty(toVertex, OUTPUT_PORT_GUIDS_ATTR , guid));; + } + } + + if(ctx.getAttribute().getRelationshipEdgeLabel().equals(INPUT_PORT_PRODUCT_EDGE_LABEL)){ + List inputPortGuids = null; + inputPortGuids = allArrayElementGuids; + toVertex.removeProperty(INPUT_PORT_GUIDS_ATTR); + if (CollectionUtils.isNotEmpty(inputPortGuids)) { + inputPortGuids.forEach(guid -> AtlasGraphUtilsV2.addEncodedProperty(toVertex, INPUT_PORT_GUIDS_ATTR , guid)); + } + } + } + RequestContext.get().endMetricRecord(metricRecorder); + } + + private void addInternalProductAttrForAppendAndRemove(AttributeMutationContext ctx, List createdElements, List deletedElements, List currentElements) { + MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("addInternalProductAttrForAppend"); + AtlasVertex toVertex = ctx.getReferringVertex(); + String toVertexType = getTypeName(toVertex); + List outputPortGuids = new ArrayList<>(); + List inputPortGuids = new ArrayList<>(); + + if (TYPE_PRODUCT.equals(toVertexType)) { + if(ctx.getAttribute().getRelationshipEdgeLabel().equals(OUTPUT_PORT_PRODUCT_EDGE_LABEL)){ + outputPortGuids = toVertex.getMultiValuedProperty(OUTPUT_PORT_GUIDS_ATTR, String.class); + + if (CollectionUtils.isNotEmpty(currentElements) && CollectionUtils.isEmpty(outputPortGuids)) { + List currentGuids = currentElements.stream() + .map(x -> ((AtlasEdge) x).getOutVertex().getProperty("__guid", String.class)) + .collect(Collectors.toList()); + + outputPortGuids.addAll(currentGuids); + } + + if (CollectionUtils.isNotEmpty(createdElements)) { + List assetGuids = createdElements.stream().map(x -> ((AtlasEdge) x).getOutVertex().getProperty("__guid", String.class)).collect(Collectors.toList()); + outputPortGuids.addAll(assetGuids); + toVertex.removeProperty(OUTPUT_PORT_GUIDS_ATTR); + if (CollectionUtils.isNotEmpty(outputPortGuids)) { + outputPortGuids.forEach(guid -> AtlasGraphUtilsV2.addEncodedProperty(toVertex, OUTPUT_PORT_GUIDS_ATTR , guid)); + } + } + + if (CollectionUtils.isNotEmpty(deletedElements)) { + List assetGuids = deletedElements.stream().map(x -> x.getOutVertex().getProperty("__guid", String.class)).collect(Collectors.toList()); + outputPortGuids.removeAll(assetGuids); + toVertex.removeProperty(OUTPUT_PORT_GUIDS_ATTR); + if (CollectionUtils.isNotEmpty(outputPortGuids)) { + outputPortGuids.forEach(guid -> AtlasGraphUtilsV2.addEncodedProperty(toVertex, OUTPUT_PORT_GUIDS_ATTR , guid)); + } + + } + } + + if (ctx.getAttribute().getRelationshipEdgeLabel().equals(INPUT_PORT_PRODUCT_EDGE_LABEL)) { + inputPortGuids = toVertex.getProperty(INPUT_PORT_GUIDS_ATTR, ArrayList.class); + if(inputPortGuids == null) { + inputPortGuids = new ArrayList<>(); + } + + if (CollectionUtils.isNotEmpty(currentElements) && CollectionUtils.isEmpty(inputPortGuids)) { + List currentGuids = currentElements.stream() + .map(x -> ((AtlasEdge) x).getOutVertex().getProperty("__guid", String.class)) + .collect(Collectors.toList()); + + inputPortGuids.addAll(currentGuids); + } + + if (CollectionUtils.isNotEmpty(createdElements)) { + List assetGuids = createdElements.stream().map(x -> ((AtlasEdge) x).getOutVertex().getProperty("__guid", String.class)).collect(Collectors.toList()); + inputPortGuids.addAll(assetGuids); + toVertex.removeProperty(INPUT_PORT_GUIDS_ATTR); + if (CollectionUtils.isNotEmpty(inputPortGuids)) { + inputPortGuids.forEach(guid -> AtlasGraphUtilsV2.addEncodedProperty(toVertex, INPUT_PORT_GUIDS_ATTR , guid)); + } + } + + if (CollectionUtils.isNotEmpty(deletedElements)) { + List assetGuids = deletedElements.stream().map(x -> x.getOutVertex().getProperty("__guid", String.class)).collect(Collectors.toList()); + inputPortGuids.removeAll(assetGuids); + toVertex.removeProperty(INPUT_PORT_GUIDS_ATTR); + if (CollectionUtils.isNotEmpty(inputPortGuids)) { + inputPortGuids.forEach(guid -> AtlasGraphUtilsV2.addEncodedProperty(toVertex, INPUT_PORT_GUIDS_ATTR , guid)); + } + } + } + } + RequestContext.get().endMetricRecord(metricRecorder); + } + private boolean shouldDeleteExistingRelations(AttributeMutationContext ctx, AtlasAttribute attribute) { boolean ret = false; AtlasEntityType entityType = typeRegistry.getEntityTypeByName(AtlasGraphUtilsV2.getTypeName(ctx.getReferringVertex())); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index 2680e482079..3dc97fa642e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -58,6 +58,8 @@ public class PreProcessorUtils { public static final String DAAP_VISIBILITY_ATTR = "daapVisibility"; public static final String DAAP_VISIBILITY_USERS_ATTR = "daapVisibilityUsers"; public static final String DAAP_VISIBILITY_GROUPS_ATTR = "daapVisibilityGroups"; + public static final String OUTPUT_PORT_GUIDS_ATTR = "daapOutputPortGuids"; + public static final String INPUT_PORT_GUIDS_ATTR = "daapInputPortGuids"; //Migration Constants public static final String MIGRATION_TYPE_PREFIX = "MIGRATION:"; From 97283f7b04cb36a9d9542dcdf292f341fd04da01 Mon Sep 17 00:00:00 2001 From: Hitesh Khandelwal Date: Fri, 31 May 2024 15:16:22 +0530 Subject: [PATCH 257/316] fixed "track_total_hits": false gives an NPE --- .../repository/graphdb/janus/AtlasElasticsearchQuery.java | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java index c3030291460..6d3ee807f77 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java @@ -445,7 +445,10 @@ private DirectIndexQueryResult getResultFromResponse(Map if (hits_0 == null) { return result; } - this.vertexTotals = (Integer) hits_0.get("total").get("value"); + LinkedHashMap approximateCount = hits_0.get("total"); + if (approximateCount != null) { + this.vertexTotals = (Integer) approximateCount.get("value"); + } List hits_1 = AtlasType.fromJson(AtlasType.toJson(hits_0.get("hits")), List.class); From b87ea5c541fcfac5a9fa7cb29641e7cdfaa7d7a2 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Fri, 31 May 2024 20:33:45 +0800 Subject: [PATCH 258/316] refactored the code --- .../store/graph/v2/EntityGraphMapper.java | 155 ++++++------------ 1 file changed, 47 insertions(+), 108 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 9b69d2e919d..12dbb5254bf 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -2008,7 +2008,7 @@ public List mapArrayValue(AttributeMutationContext ctx, EntityMutationContext co case INPUT_PORT_PRODUCT_EDGE_LABEL: case OUTPUT_PORT_PRODUCT_EDGE_LABEL: - addInternalProductAttrForAppendAndRemove(ctx, newElementsCreated, removedElements, currentElements); + addInternalProductAttr(ctx, newElementsCreated, removedElements, currentElements); break; } @@ -2036,15 +2036,9 @@ public List appendArrayValue(AttributeMutationContext ctx, EntityMutationContext AtlasAttribute inverseRefAttribute = attribute.getInverseRefAttribute(); Cardinality cardinality = attribute.getAttributeDef().getCardinality(); List newElementsCreated = new ArrayList<>(); - List currentElements; boolean isNewElementsNull = newElements == null; - if (isReference && !isSoftReference) { - currentElements = (List) getCollectionElementsUsingRelationship(ctx.getReferringVertex(), attribute, isStructType); - } else { - currentElements = (List) getArrayElementsProperty(elementType, isSoftReference, ctx.getReferringVertex(), ctx.getVertexProperty()); - } if (isNewElementsNull) { newElements = new ArrayList(); @@ -2105,7 +2099,7 @@ public List appendArrayValue(AttributeMutationContext ctx, EntityMutationContext case INPUT_PORT_PRODUCT_EDGE_LABEL: case OUTPUT_PORT_PRODUCT_EDGE_LABEL: - addInternalProductAttrForAppendAndRemove(ctx, newElementsCreated, new ArrayList<>(0), currentElements); + addInternalProductAttr(ctx, newElementsCreated, new ArrayList<>(0), null); break; } @@ -2130,18 +2124,10 @@ public List removeArrayValue(AttributeMutationContext ctx, EntityMutationContext Cardinality cardinality = attribute.getAttributeDef().getCardinality(); List removedElements = new ArrayList<>(); List entityRelationsDeleted = new ArrayList<>(); - boolean isReference = isReference(elementType); - boolean isSoftReference = ctx.getAttribute().getAttributeDef().isSoftReferenced(); - List currentElements; boolean isNewElementsNull = elementsDeleted == null; - if (isReference && !isSoftReference) { - currentElements = (List) getCollectionElementsUsingRelationship(ctx.getReferringVertex(), attribute, isStructType); - } else { - currentElements = (List) getArrayElementsProperty(elementType, isSoftReference, ctx.getReferringVertex(), ctx.getVertexProperty()); - } if (isNewElementsNull) { elementsDeleted = new ArrayList(); @@ -2186,7 +2172,7 @@ public List removeArrayValue(AttributeMutationContext ctx, EntityMutationContext case INPUT_PORT_PRODUCT_EDGE_LABEL: case OUTPUT_PORT_PRODUCT_EDGE_LABEL: - addInternalProductAttrForAppendAndRemove(ctx, new ArrayList<>(0) , removedElements, currentElements); + addInternalProductAttr(ctx, new ArrayList<>(0) , removedElements, null); break; } @@ -2225,118 +2211,71 @@ private void addEdgesToContext(String guid, List newElementsCreated, Lis } } - private void addInternalProductAttr (AttributeMutationContext ctx, List allArrayElements, List deletedElements) { - MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("addInternalProductAttr"); + private void addInternalProductAttr(AttributeMutationContext ctx, List createdElements, List deletedElements, List currentElements) throws AtlasBaseException { + MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("addInternalProductAttrForAppend"); AtlasVertex toVertex = ctx.getReferringVertex(); String toVertexType = getTypeName(toVertex); + List portGuids = new ArrayList<>(); if (TYPE_PRODUCT.equals(toVertexType)) { - List allArrayElementGuids = allArrayElements.stream() - .map(x -> ((AtlasEdge) x).getOutVertex().getProperty("__guid", String.class)) - .collect(Collectors.toList()); - // Removing deleted elements from allArrayElementGuids - if (CollectionUtils.isNotEmpty(deletedElements)) { - List deletedGuids = deletedElements.stream() - .map(x -> x.getOutVertex().getProperty("__guid", String.class)) - .collect(Collectors.toList()); - allArrayElementGuids.removeAll(deletedGuids); + if(currentElements == null){ + AtlasAttribute attribute = ctx.getAttribute(); + AtlasArrayType arrType = (AtlasArrayType) attribute.getAttributeType(); + AtlasType elementType = arrType.getElementType(); + boolean isStructType = (TypeCategory.STRUCT == elementType.getTypeCategory()) || + (TypeCategory.STRUCT == attribute.getDefinedInType().getTypeCategory()); + boolean isReference = isReference(elementType); + boolean isSoftReference = ctx.getAttribute().getAttributeDef().isSoftReferenced(); + + if (isReference && !isSoftReference) { + currentElements = (List) getCollectionElementsUsingRelationship(ctx.getReferringVertex(), attribute, isStructType); + } else { + currentElements = (List) getArrayElementsProperty(elementType, isSoftReference, ctx.getReferringVertex(), ctx.getVertexProperty()); + } } if(ctx.getAttribute().getRelationshipEdgeLabel().equals(OUTPUT_PORT_PRODUCT_EDGE_LABEL)){ - List outputPortGuids = null; - outputPortGuids = allArrayElementGuids; - toVertex.removeProperty(OUTPUT_PORT_GUIDS_ATTR); - if (CollectionUtils.isNotEmpty(outputPortGuids)) { - outputPortGuids.forEach(guid -> AtlasGraphUtilsV2.addEncodedProperty(toVertex, OUTPUT_PORT_GUIDS_ATTR , guid));; - } + addOrRemoveInternalAttr(toVertex, OUTPUT_PORT_GUIDS_ATTR, createdElements, currentElements, deletedElements, portGuids); } - - if(ctx.getAttribute().getRelationshipEdgeLabel().equals(INPUT_PORT_PRODUCT_EDGE_LABEL)){ - List inputPortGuids = null; - inputPortGuids = allArrayElementGuids; - toVertex.removeProperty(INPUT_PORT_GUIDS_ATTR); - if (CollectionUtils.isNotEmpty(inputPortGuids)) { - inputPortGuids.forEach(guid -> AtlasGraphUtilsV2.addEncodedProperty(toVertex, INPUT_PORT_GUIDS_ATTR , guid)); - } + if (ctx.getAttribute().getRelationshipEdgeLabel().equals(INPUT_PORT_PRODUCT_EDGE_LABEL)) { + addOrRemoveInternalAttr(toVertex, INPUT_PORT_GUIDS_ATTR, createdElements, currentElements, deletedElements, portGuids); } + }else{ + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "InternalProductAttribute can only be added to Product entity"); } RequestContext.get().endMetricRecord(metricRecorder); } - private void addInternalProductAttrForAppendAndRemove(AttributeMutationContext ctx, List createdElements, List deletedElements, List currentElements) { - MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("addInternalProductAttrForAppend"); - AtlasVertex toVertex = ctx.getReferringVertex(); - String toVertexType = getTypeName(toVertex); - List outputPortGuids = new ArrayList<>(); - List inputPortGuids = new ArrayList<>(); - - if (TYPE_PRODUCT.equals(toVertexType)) { - if(ctx.getAttribute().getRelationshipEdgeLabel().equals(OUTPUT_PORT_PRODUCT_EDGE_LABEL)){ - outputPortGuids = toVertex.getMultiValuedProperty(OUTPUT_PORT_GUIDS_ATTR, String.class); - - if (CollectionUtils.isNotEmpty(currentElements) && CollectionUtils.isEmpty(outputPortGuids)) { - List currentGuids = currentElements.stream() - .map(x -> ((AtlasEdge) x).getOutVertex().getProperty("__guid", String.class)) - .collect(Collectors.toList()); + private void addOrRemoveInternalAttr(AtlasVertex toVertex, String internalAttr, List createdElements, List currentElements, List deletedElements, List portGuids) { + portGuids = toVertex.getMultiValuedProperty(internalAttr, String.class); - outputPortGuids.addAll(currentGuids); - } - - if (CollectionUtils.isNotEmpty(createdElements)) { - List assetGuids = createdElements.stream().map(x -> ((AtlasEdge) x).getOutVertex().getProperty("__guid", String.class)).collect(Collectors.toList()); - outputPortGuids.addAll(assetGuids); - toVertex.removeProperty(OUTPUT_PORT_GUIDS_ATTR); - if (CollectionUtils.isNotEmpty(outputPortGuids)) { - outputPortGuids.forEach(guid -> AtlasGraphUtilsV2.addEncodedProperty(toVertex, OUTPUT_PORT_GUIDS_ATTR , guid)); - } - } + if (CollectionUtils.isNotEmpty(currentElements) && CollectionUtils.isEmpty(portGuids)) { + List currentGuids = currentElements.stream() + .map(x -> ((AtlasEdge) x).getOutVertex().getProperty("__guid", String.class)) + .collect(Collectors.toList()); - if (CollectionUtils.isNotEmpty(deletedElements)) { - List assetGuids = deletedElements.stream().map(x -> x.getOutVertex().getProperty("__guid", String.class)).collect(Collectors.toList()); - outputPortGuids.removeAll(assetGuids); - toVertex.removeProperty(OUTPUT_PORT_GUIDS_ATTR); - if (CollectionUtils.isNotEmpty(outputPortGuids)) { - outputPortGuids.forEach(guid -> AtlasGraphUtilsV2.addEncodedProperty(toVertex, OUTPUT_PORT_GUIDS_ATTR , guid)); - } + portGuids.addAll(currentGuids); + } - } + if (CollectionUtils.isNotEmpty(createdElements)) { + List assetGuids = createdElements.stream().map(x -> ((AtlasEdge) x).getOutVertex().getProperty("__guid", String.class)).collect(Collectors.toList()); + portGuids.addAll(assetGuids); + toVertex.removeProperty(internalAttr); + if (CollectionUtils.isNotEmpty(portGuids)) { + portGuids.forEach(guid -> AtlasGraphUtilsV2.addEncodedProperty(toVertex, internalAttr , guid)); } + } - if (ctx.getAttribute().getRelationshipEdgeLabel().equals(INPUT_PORT_PRODUCT_EDGE_LABEL)) { - inputPortGuids = toVertex.getProperty(INPUT_PORT_GUIDS_ATTR, ArrayList.class); - if(inputPortGuids == null) { - inputPortGuids = new ArrayList<>(); - } - - if (CollectionUtils.isNotEmpty(currentElements) && CollectionUtils.isEmpty(inputPortGuids)) { - List currentGuids = currentElements.stream() - .map(x -> ((AtlasEdge) x).getOutVertex().getProperty("__guid", String.class)) - .collect(Collectors.toList()); - - inputPortGuids.addAll(currentGuids); - } - - if (CollectionUtils.isNotEmpty(createdElements)) { - List assetGuids = createdElements.stream().map(x -> ((AtlasEdge) x).getOutVertex().getProperty("__guid", String.class)).collect(Collectors.toList()); - inputPortGuids.addAll(assetGuids); - toVertex.removeProperty(INPUT_PORT_GUIDS_ATTR); - if (CollectionUtils.isNotEmpty(inputPortGuids)) { - inputPortGuids.forEach(guid -> AtlasGraphUtilsV2.addEncodedProperty(toVertex, INPUT_PORT_GUIDS_ATTR , guid)); - } - } - - if (CollectionUtils.isNotEmpty(deletedElements)) { - List assetGuids = deletedElements.stream().map(x -> x.getOutVertex().getProperty("__guid", String.class)).collect(Collectors.toList()); - inputPortGuids.removeAll(assetGuids); - toVertex.removeProperty(INPUT_PORT_GUIDS_ATTR); - if (CollectionUtils.isNotEmpty(inputPortGuids)) { - inputPortGuids.forEach(guid -> AtlasGraphUtilsV2.addEncodedProperty(toVertex, INPUT_PORT_GUIDS_ATTR , guid)); - } - } + if (CollectionUtils.isNotEmpty(deletedElements)) { + List assetGuids = deletedElements.stream().map(x -> x.getOutVertex().getProperty("__guid", String.class)).collect(Collectors.toList()); + portGuids.removeAll(assetGuids); + toVertex.removeProperty(internalAttr); + if (CollectionUtils.isNotEmpty(portGuids)) { + portGuids.forEach(guid -> AtlasGraphUtilsV2.addEncodedProperty(toVertex, internalAttr , guid)); } + } - RequestContext.get().endMetricRecord(metricRecorder); } private boolean shouldDeleteExistingRelations(AttributeMutationContext ctx, AtlasAttribute attribute) { From d63a019932d2285b333ee8349b2e5f828a238ff3 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Fri, 31 May 2024 22:47:35 +0800 Subject: [PATCH 259/316] error message changed --- .../atlas/repository/store/graph/v2/EntityGraphMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 12dbb5254bf..d67e5f9fa0c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -2242,7 +2242,7 @@ private void addInternalProductAttr(AttributeMutationContext ctx, List c addOrRemoveInternalAttr(toVertex, INPUT_PORT_GUIDS_ATTR, createdElements, currentElements, deletedElements, portGuids); } }else{ - throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "InternalProductAttribute can only be added to Product entity"); + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Can not update product relations while updating any asset"); } RequestContext.get().endMetricRecord(metricRecorder); } From c1c23166430ba482957562d15095edb9eaff3602 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Sat, 1 Jun 2024 00:17:52 +0800 Subject: [PATCH 260/316] removed duplicate code --- .../store/graph/v2/EntityGraphMapper.java | 20 +++++-------------- 1 file changed, 5 insertions(+), 15 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index d67e5f9fa0c..0e911911934 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -2225,14 +2225,8 @@ private void addInternalProductAttr(AttributeMutationContext ctx, List c AtlasType elementType = arrType.getElementType(); boolean isStructType = (TypeCategory.STRUCT == elementType.getTypeCategory()) || (TypeCategory.STRUCT == attribute.getDefinedInType().getTypeCategory()); - boolean isReference = isReference(elementType); - boolean isSoftReference = ctx.getAttribute().getAttributeDef().isSoftReferenced(); - if (isReference && !isSoftReference) { - currentElements = (List) getCollectionElementsUsingRelationship(ctx.getReferringVertex(), attribute, isStructType); - } else { - currentElements = (List) getArrayElementsProperty(elementType, isSoftReference, ctx.getReferringVertex(), ctx.getVertexProperty()); - } + currentElements = (List) getCollectionElementsUsingRelationship(ctx.getReferringVertex(), attribute, isStructType); } if(ctx.getAttribute().getRelationshipEdgeLabel().equals(OUTPUT_PORT_PRODUCT_EDGE_LABEL)){ @@ -2261,20 +2255,16 @@ private void addOrRemoveInternalAttr(AtlasVertex toVertex, String internalAttr, if (CollectionUtils.isNotEmpty(createdElements)) { List assetGuids = createdElements.stream().map(x -> ((AtlasEdge) x).getOutVertex().getProperty("__guid", String.class)).collect(Collectors.toList()); portGuids.addAll(assetGuids); - toVertex.removeProperty(internalAttr); - if (CollectionUtils.isNotEmpty(portGuids)) { - portGuids.forEach(guid -> AtlasGraphUtilsV2.addEncodedProperty(toVertex, internalAttr , guid)); - } } if (CollectionUtils.isNotEmpty(deletedElements)) { List assetGuids = deletedElements.stream().map(x -> x.getOutVertex().getProperty("__guid", String.class)).collect(Collectors.toList()); portGuids.removeAll(assetGuids); - toVertex.removeProperty(internalAttr); - if (CollectionUtils.isNotEmpty(portGuids)) { - portGuids.forEach(guid -> AtlasGraphUtilsV2.addEncodedProperty(toVertex, internalAttr , guid)); - } + } + toVertex.removeProperty(internalAttr); + if (CollectionUtils.isNotEmpty(portGuids)) { + portGuids.forEach(guid -> AtlasGraphUtilsV2.addEncodedProperty(toVertex, internalAttr , guid)); } } From e15f885f447183bc2291f0a2cf7715cf5fde0164 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Sat, 1 Jun 2024 00:39:24 +0800 Subject: [PATCH 261/316] fix: cariable override --- .../repository/store/graph/v2/EntityGraphMapper.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 0e911911934..208cf1b7335 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -2215,7 +2215,6 @@ private void addInternalProductAttr(AttributeMutationContext ctx, List c MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("addInternalProductAttrForAppend"); AtlasVertex toVertex = ctx.getReferringVertex(); String toVertexType = getTypeName(toVertex); - List portGuids = new ArrayList<>(); if (TYPE_PRODUCT.equals(toVertexType)) { @@ -2230,10 +2229,10 @@ private void addInternalProductAttr(AttributeMutationContext ctx, List c } if(ctx.getAttribute().getRelationshipEdgeLabel().equals(OUTPUT_PORT_PRODUCT_EDGE_LABEL)){ - addOrRemoveInternalAttr(toVertex, OUTPUT_PORT_GUIDS_ATTR, createdElements, currentElements, deletedElements, portGuids); + addOrRemoveInternalAttr(toVertex, OUTPUT_PORT_GUIDS_ATTR, createdElements, currentElements, deletedElements); } if (ctx.getAttribute().getRelationshipEdgeLabel().equals(INPUT_PORT_PRODUCT_EDGE_LABEL)) { - addOrRemoveInternalAttr(toVertex, INPUT_PORT_GUIDS_ATTR, createdElements, currentElements, deletedElements, portGuids); + addOrRemoveInternalAttr(toVertex, INPUT_PORT_GUIDS_ATTR, createdElements, currentElements, deletedElements); } }else{ throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Can not update product relations while updating any asset"); @@ -2241,8 +2240,8 @@ private void addInternalProductAttr(AttributeMutationContext ctx, List c RequestContext.get().endMetricRecord(metricRecorder); } - private void addOrRemoveInternalAttr(AtlasVertex toVertex, String internalAttr, List createdElements, List currentElements, List deletedElements, List portGuids) { - portGuids = toVertex.getMultiValuedProperty(internalAttr, String.class); + private void addOrRemoveInternalAttr(AtlasVertex toVertex, String internalAttr, List createdElements, List currentElements, List deletedElements) { + List portGuids = toVertex.getMultiValuedProperty(internalAttr, String.class); if (CollectionUtils.isNotEmpty(currentElements) && CollectionUtils.isEmpty(portGuids)) { List currentGuids = currentElements.stream() From 4b7217de8f4be4a1f87da30280ff7ff55417e101 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Sat, 1 Jun 2024 01:00:06 +0800 Subject: [PATCH 262/316] union operation for createdElements --- .../atlas/repository/store/graph/v2/EntityGraphMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 208cf1b7335..f18d802ffc2 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -2253,7 +2253,7 @@ private void addOrRemoveInternalAttr(AtlasVertex toVertex, String internalAttr, if (CollectionUtils.isNotEmpty(createdElements)) { List assetGuids = createdElements.stream().map(x -> ((AtlasEdge) x).getOutVertex().getProperty("__guid", String.class)).collect(Collectors.toList()); - portGuids.addAll(assetGuids); + portGuids = (List) CollectionUtils.union(portGuids, assetGuids); } if (CollectionUtils.isNotEmpty(deletedElements)) { From afaec7aedca87094d616a2d43233d05bf8ed3efe Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Mon, 3 Jun 2024 20:16:46 +0530 Subject: [PATCH 263/316] fix: add check only for table not to check for supertypes --- .../main/java/org/apache/atlas/type/AtlasTypeRegistry.java | 3 +++ .../atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java | 7 ++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java index 4cb7e4a185f..82628ce73e5 100644 --- a/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java +++ b/intg/src/main/java/org/apache/atlas/type/AtlasTypeRegistry.java @@ -44,6 +44,9 @@ public class AtlasTypeRegistry { private static final Logger LOG = LoggerFactory.getLogger(AtlasTypeRegistry.class); private static final int DEFAULT_LOCK_MAX_WAIT_TIME_IN_SECONDS = 15; + public static final ArrayList TYPENAMES_TO_SKIP_SUPER_TYPE_CHECK = new ArrayList() {{ + add("Table"); + }}; protected RegistryData registryData; private final TypeRegistryUpdateSynchronizer updateSynchronizer; private final Set missingRelationshipDefs; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java index 5f97d5645cd..d6af87c2247 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java @@ -35,8 +35,9 @@ import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasEnumType; import org.apache.atlas.type.AtlasStructType; -import org.apache.atlas.type.AtlasStructType.AtlasAttribute; +import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.type.AtlasType; +import org.apache.atlas.type.AtlasStructType.AtlasAttribute; import org.apache.atlas.util.FileUtils; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.atlas.utils.AtlasPerfMetrics.MetricRecorder; @@ -349,14 +350,14 @@ public static AtlasVertex findByUniqueAttributes(AtlasGraph graph, AtlasEntityTy vertex = findByTypeAndUniquePropertyName(graph, typeName, uniqAttrValues); // if no instance of given typeName is found, try to find an instance of type's sub-type - if (vertex == null && !entitySubTypes.isEmpty()) { + if (vertex == null && !entitySubTypes.isEmpty() && !AtlasTypeRegistry.TYPENAMES_TO_SKIP_SUPER_TYPE_CHECK.contains(typeName)) { vertex = findBySuperTypeAndUniquePropertyName(graph, typeName, uniqAttrValues); } } else { vertex = findByTypeAndPropertyName(graph, typeName, attrNameValues); // if no instance of given typeName is found, try to find an instance of type's sub-type - if (vertex == null && !entitySubTypes.isEmpty()) { + if (vertex == null && !entitySubTypes.isEmpty() && !AtlasTypeRegistry.TYPENAMES_TO_SKIP_SUPER_TYPE_CHECK.contains(typeName)) { vertex = findBySuperTypeAndPropertyName(graph, typeName, attrNameValues); } } From c004d353c196b45d162792043a728821a65f0e95 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Mon, 3 Jun 2024 20:20:43 +0530 Subject: [PATCH 264/316] nit: add comment for better generalising --- .../atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java index d6af87c2247..42d30d39ca4 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasGraphUtilsV2.java @@ -350,6 +350,7 @@ public static AtlasVertex findByUniqueAttributes(AtlasGraph graph, AtlasEntityTy vertex = findByTypeAndUniquePropertyName(graph, typeName, uniqAttrValues); // if no instance of given typeName is found, try to find an instance of type's sub-type + // Added exception for few types to solve https://atlanhq.atlassian.net/browse/PLT-1638 if (vertex == null && !entitySubTypes.isEmpty() && !AtlasTypeRegistry.TYPENAMES_TO_SKIP_SUPER_TYPE_CHECK.contains(typeName)) { vertex = findBySuperTypeAndUniquePropertyName(graph, typeName, uniqAttrValues); } @@ -357,6 +358,7 @@ public static AtlasVertex findByUniqueAttributes(AtlasGraph graph, AtlasEntityTy vertex = findByTypeAndPropertyName(graph, typeName, attrNameValues); // if no instance of given typeName is found, try to find an instance of type's sub-type + // Added exception for few types to solve https://atlanhq.atlassian.net/browse/PLT-1638 if (vertex == null && !entitySubTypes.isEmpty() && !AtlasTypeRegistry.TYPENAMES_TO_SKIP_SUPER_TYPE_CHECK.contains(typeName)) { vertex = findBySuperTypeAndPropertyName(graph, typeName, attrNameValues); } From f38a52a47801c0930516cfda632b1a330cf33061 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Tue, 4 Jun 2024 11:48:36 +0530 Subject: [PATCH 265/316] feat: rebuild alias endpoint --- .../store/aliasstore/ESAliasStore.java | 28 +++++++++++++++++++ .../store/graph/AtlasEntityStore.java | 2 ++ .../store/graph/v2/AtlasEntityStoreV2.java | 10 +++++++ .../org/apache/atlas/web/rest/EntityREST.java | 7 +++++ 4 files changed, 47 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/aliasstore/ESAliasStore.java b/repository/src/main/java/org/apache/atlas/repository/store/aliasstore/ESAliasStore.java index 1991106b965..88fdf0f4af3 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/aliasstore/ESAliasStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/aliasstore/ESAliasStore.java @@ -141,6 +141,23 @@ public boolean updateAlias(AtlasEntity.AtlasEntityWithExtInfo accessControl, Atl return true; } + public void rebuildAlias(AtlasEntity.AtlasEntityWithExtInfo accessControl) throws AtlasBaseException { + String aliasName = getAliasName(accessControl.getEntity()); + + Map filter; + + if (PERSONA_ENTITY_TYPE.equals(accessControl.getEntity().getTypeName())) { + filter = getFilterForPersona(accessControl); + } else { + filter = getFilterForPurpose(accessControl.getEntity()); + } + + ESAliasRequestBuilder requestBuilder = new ESAliasRequestBuilder(); + requestBuilder.addAction(ADD, new AliasAction(getIndexNameFromAliasIfExists(VERTEX_INDEX_NAME), aliasName, filter)); + + graph.createOrUpdateESAlias(requestBuilder); + } + @Override public boolean deleteAlias(String aliasName) throws AtlasBaseException { graph.deleteESAlias(getIndexNameFromAliasIfExists(VERTEX_INDEX_NAME), aliasName); @@ -161,6 +178,17 @@ private Map getFilterForPersona(AtlasEntity.AtlasEntityWithExtIn return esClausesToFilter(allowClauseList); } + private Map getFilterForPersona(AtlasEntity.AtlasEntityWithExtInfo persona) throws AtlasBaseException { + List> allowClauseList = new ArrayList<>(); + + List policies = getPolicies(persona); + if (CollectionUtils.isNotEmpty(policies)) { + personaPolicyToESDslClauses(policies, allowClauseList); + } + + return esClausesToFilter(allowClauseList); + } + private Map getFilterForPurpose(AtlasEntity purpose) throws AtlasBaseException { List> allowClauseList = new ArrayList<>(); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java index 960ec3808e4..d072139aef5 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java @@ -364,4 +364,6 @@ EntityMutationResponse deleteByUniqueAttributes(List objectIds) void repairMeaningAttributeForTerms(List termGuids) throws AtlasBaseException; + public void repairAlias(String guid) throws AtlasBaseException; + } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 2d6d28c0a72..77d530307e8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -49,6 +49,7 @@ import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.patches.PatchContext; import org.apache.atlas.repository.patches.ReIndexPatch; +import org.apache.atlas.repository.store.aliasstore.ESAliasStore; import org.apache.atlas.repository.store.graph.AtlasEntityStore; import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; import org.apache.atlas.repository.store.graph.EntityGraphDiscovery; @@ -147,6 +148,8 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore { private final AtlasRelationshipStore atlasRelationshipStore; private final FeatureFlagStore featureFlagStore; + private final ESAliasStore esAliasStore; + @Inject public AtlasEntityStoreV2(AtlasGraph graph, DeleteHandlerDelegate deleteDelegate, RestoreHandlerV1 restoreHandlerV1, AtlasTypeRegistry typeRegistry, IAtlasEntityChangeNotifier entityChangeNotifier, EntityGraphMapper entityGraphMapper, TaskManagement taskManagement, @@ -163,6 +166,7 @@ public AtlasEntityStoreV2(AtlasGraph graph, DeleteHandlerDelegate deleteDelegate this.taskManagement = taskManagement; this.atlasRelationshipStore = atlasRelationshipStore; this.featureFlagStore = featureFlagStore; + this.esAliasStore = new ESAliasStore(graph, entityRetriever); try { this.discovery = new EntityDiscoveryService(typeRegistry, graph, null, null, null, null); @@ -2703,6 +2707,12 @@ private void repairMeanings(AtlasVertex assetVertex) { } } + @Override + public void repairAlias(String guid) throws AtlasBaseException { + // Fetch entity with extenfo + AtlasEntity.AtlasEntityWithExtInfo entity = entityRetriever.toAtlasEntityWithExtInfo(guid); + this.esAliasStore.rebuildAlias(entity); + } } diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java index 0ebb48b16c6..31283924a23 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java @@ -1930,4 +1930,11 @@ public void repairIndexByTypeName(@PathParam("typename") String typename, @Query AtlasPerfTracer.log(perf); } } + + @POST + @Path("/repairalias/{guid}") + @Timed + public void repairAlias(@PathParam("guid") String guid) throws AtlasBaseException { + entitiesStore.repairAlias(guid); + } } From 6d26361ac12e9208e72415d90c63987f78e0a754 Mon Sep 17 00:00:00 2001 From: arpit-at Date: Tue, 4 Jun 2024 18:26:03 +0530 Subject: [PATCH 266/316] DG-1374 Update in pre-processor to handle the admin attributes --- .../java/org/apache/atlas/AtlasErrorCode.java | 2 +- .../preprocessor/ConnectionPreProcessor.java | 110 ++++++++++++------ 2 files changed, 73 insertions(+), 39 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java index 457f00e562e..1ad37b25649 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java +++ b/intg/src/main/java/org/apache/atlas/AtlasErrorCode.java @@ -291,7 +291,7 @@ public enum AtlasErrorCode { TASK_TYPE_NOT_SUPPORTED(400, "ATLAS-400-00-112", "Task type {0} is not supported"), PERSONA_POLICY_ASSETS_LIMIT_EXCEEDED(400, "ATLAS-400-00-113", "Exceeded limit of maximum allowed assets across policies for a Persona: Limit: {0}, assets: {1}"), - ADMIN_LIST_SHOULD_NOT_BE_EMPTY(400, "ATLAS-400-00-113", "Admin list should not be empty for type {0}"); + ADMIN_LIST_SHOULD_NOT_BE_EMPTY(400, "ATLAS-400-00-114", "Admin list should not be empty for type {0}"); private String errorCode; private String errorMessage; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java index 1ffb7b26b62..c8ff645151a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java @@ -17,6 +17,7 @@ */ package org.apache.atlas.repository.store.graph.v2.preprocessor; +import com.google.common.collect.Sets; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.DeleteType; import org.apache.atlas.RequestContext; @@ -48,18 +49,15 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.stream.Collectors; +import java.util.stream.Stream; import static org.apache.atlas.authorize.AtlasAuthorizerFactory.ATLAS_AUTHORIZER_IMPL; import static org.apache.atlas.authorize.AtlasAuthorizerFactory.CURRENT_AUTHORIZER_IMPL; import static org.apache.atlas.repository.Constants.ATTR_ADMIN_GROUPS; import static org.apache.atlas.repository.Constants.ATTR_ADMIN_ROLES; import static org.apache.atlas.repository.Constants.ATTR_ADMIN_USERS; -import static org.apache.atlas.repository.Constants.CREATED_BY_KEY; import static org.apache.atlas.repository.Constants.POLICY_ENTITY_TYPE; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; @@ -155,60 +153,96 @@ private void processCreateConnection(AtlasStruct struct) throws AtlasBaseExcepti } } - private void processUpdateConnection(EntityMutationContext context, - AtlasStruct entity) throws AtlasBaseException { - + private void processUpdateConnection(EntityMutationContext context, AtlasStruct entity) throws AtlasBaseException { AtlasEntity connection = (AtlasEntity) entity; - if (ATLAS_AUTHORIZER_IMPL.equalsIgnoreCase(CURRENT_AUTHORIZER_IMPL)) { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdateConnection"); - AtlasVertex vertex = context.getVertex(connection.getGuid()); AtlasEntity existingConnEntity = entityRetriever.toAtlasEntity(vertex); - String roleName = String.format(CONN_NAME_PATTERN, connection.getGuid()); - String vertexQName = vertex.getProperty(QUALIFIED_NAME, String.class); entity.setAttribute(QUALIFIED_NAME, vertexQName); - RoleRepresentation representation = getKeycloakClient().getRoleByName(roleName); - // String creatorUser = vertex.getProperty(CREATED_BY_KEY, String.class); - - if (connection.hasAttribute(ATTR_ADMIN_USERS)) { - List newAdminUsers = (List) connection.getAttribute(ATTR_ADMIN_USERS); - List currentAdminUsers = (List) existingConnEntity.getAttribute(ATTR_ADMIN_USERS); - if (CollectionUtils.isEmpty(newAdminUsers)) { - throw new AtlasBaseException(AtlasErrorCode.ADMIN_LIST_SHOULD_NOT_BE_EMPTY, connection.getTypeName()); - } - - connection.setAttribute(ATTR_ADMIN_USERS, newAdminUsers); - if (CollectionUtils.isNotEmpty(newAdminUsers) || CollectionUtils.isNotEmpty(currentAdminUsers)) { - keycloakStore.updateRoleUsers(roleName, currentAdminUsers, newAdminUsers, representation); - } + //optional is used here to distinguish if the admin related attributes are set in request body or not (else part) + //if set, check for empty list so that appropriate error can be thrown + List newAdminUsers = getAttributeList(connection, ATTR_ADMIN_USERS).orElse(null); + List currentAdminUsers = getAttributeList(existingConnEntity, ATTR_ADMIN_USERS).orElseGet(ArrayList::new); + + List newAdminGroups = getAttributeList(connection, ATTR_ADMIN_GROUPS).orElse(null); + List currentAdminGroups = getAttributeList(existingConnEntity, ATTR_ADMIN_GROUPS).orElseGet(ArrayList::new); + + List newAdminRoles = getAttributeList(connection, ATTR_ADMIN_ROLES).orElse(null); + List currentAdminRoles = getAttributeList(existingConnEntity, ATTR_ADMIN_ROLES).orElseGet(ArrayList::new); + + // Check conditions and throw exceptions as necessary + if (newAdminUsers == null && newAdminGroups == null && newAdminRoles == null) { + // If all new admin attributes are null, no action required as these are not meant to update in the request + RequestContext.get().endMetricRecord(metricRecorder); + return; } - if (connection.hasAttribute(ATTR_ADMIN_GROUPS)) { - List newAdminGroups = (List) connection.getAttribute(ATTR_ADMIN_GROUPS); - List currentAdminGroups = (List) existingConnEntity.getAttribute(ATTR_ADMIN_GROUPS); + // Check if any new admin attribute list is empty + boolean emptyName = newAdminUsers != null && newAdminUsers.isEmpty(); + boolean emptyGroup = newAdminGroups != null && newAdminGroups.isEmpty(); + boolean emptyRole = newAdminRoles != null && newAdminRoles.isEmpty(); - if (CollectionUtils.isNotEmpty(newAdminGroups) || CollectionUtils.isNotEmpty(currentAdminGroups)) { - keycloakStore.updateRoleGroups(roleName, currentAdminGroups, newAdminGroups, representation); - } + // Throw exception if all new admin attributes are empty + if (emptyName && emptyGroup && emptyRole) { + throw new AtlasBaseException(AtlasErrorCode.ADMIN_LIST_SHOULD_NOT_BE_EMPTY, existingConnEntity.getTypeName()); } - if (connection.hasAttribute(ATTR_ADMIN_ROLES)) { - List newAdminRoles = (List) connection.getAttribute(ATTR_ADMIN_ROLES); - List currentAdminRoles = (List) existingConnEntity.getAttribute(ATTR_ADMIN_ROLES); + //calculate final state for all admin attributes if all 3 are empty at the end throw exception now + List finalStateUsers = emptyName ? new ArrayList<>() : newAdminUsers; + List finalStateRoles = emptyRole ? new ArrayList<>() : newAdminRoles; + List finalStateGroups = emptyGroup ? new ArrayList<>() : newAdminGroups; - if (CollectionUtils.isNotEmpty(newAdminRoles) || CollectionUtils.isNotEmpty(currentAdminRoles)) { - keycloakStore.updateRoleRoles(roleName, currentAdminRoles, newAdminRoles, representation); - } + if (CollectionUtils.isNotEmpty(finalStateUsers) && CollectionUtils.isEmpty(finalStateGroups) && CollectionUtils.isEmpty(finalStateRoles)) { + throw new AtlasBaseException(AtlasErrorCode.ADMIN_LIST_SHOULD_NOT_BE_EMPTY, existingConnEntity.getTypeName()); } + // Update Keycloak roles + RoleRepresentation representation = getKeycloakClient().getRoleByName(roleName); + updateKeycloakRoleUsers(roleName, currentAdminUsers, newAdminUsers != null ? newAdminUsers : new ArrayList<>(), representation); + updateKeycloakRoleGroups(roleName, currentAdminGroups, newAdminGroups != null ? newAdminGroups : new ArrayList<>(), representation); + updateKeycloakRoleRoles(roleName, currentAdminRoles, newAdminRoles != null ? newAdminRoles : new ArrayList<>(), representation); + RequestContext.get().endMetricRecord(metricRecorder); } } + + private Optional> getAttributeList(AtlasEntity entity, String attributeName) { + if (entity.hasAttribute(attributeName)) { + return Optional.of((List) entity.getAttribute(attributeName)); + } + return Optional.empty(); + } + + // Check if all lists are empty + private boolean areAllListEmpty(List... lists) { + return Arrays.stream(lists).allMatch(Collection::isEmpty); + } + + private void updateKeycloakRoleUsers(String roleName, List currentUsers, List newUsers, RoleRepresentation representation) throws AtlasBaseException { + if (!newUsers.isEmpty() || !currentUsers.isEmpty()) { + keycloakStore.updateRoleUsers(roleName, currentUsers, newUsers, representation); + } + } + + private void updateKeycloakRoleGroups(String roleName, List currentGroups, List newGroups, RoleRepresentation representation) throws AtlasBaseException { + if (!newGroups.isEmpty() || !currentGroups.isEmpty()) { + keycloakStore.updateRoleGroups(roleName, currentGroups, newGroups, representation); + } + } + + private void updateKeycloakRoleRoles(String roleName, List currentRoles, List newRoles, RoleRepresentation representation) throws AtlasBaseException { + if (!newRoles.isEmpty() || !currentRoles.isEmpty()) { + keycloakStore.updateRoleRoles(roleName, currentRoles, newRoles, representation); + } + } + + + @Override public void processDelete(AtlasVertex vertex) throws AtlasBaseException { // Process Delete connection role and policies in case of hard delete or purge From 3cd1e03b860a8f3b4af1ea36e9e1f13b50675041 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Tue, 4 Jun 2024 19:16:35 +0530 Subject: [PATCH 267/316] feat: add more checks --- .../store/graph/v2/AtlasEntityStoreV2.java | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 77d530307e8..846510c9ea4 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -2709,8 +2709,25 @@ private void repairMeanings(AtlasVertex assetVertex) { } @Override public void repairAlias(String guid) throws AtlasBaseException { - // Fetch entity with extenfo + // Fetch entity with extInfo AtlasEntity.AtlasEntityWithExtInfo entity = entityRetriever.toAtlasEntityWithExtInfo(guid); + + if (entity == null) { + throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guid); + } + + // Validate entity status + if (entity.getEntity().getStatus() != ACTIVE) { + throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_DELETED, guid); + } + + // Validate entity type + String entityType = entity.getEntity().getTypeName(); + if (!PERSONA_ENTITY_TYPE.equals(entityType)) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_OBJECT_ID, entityType); + } + + // Rebuild alias this.esAliasStore.rebuildAlias(entity); } } From 269d78c9a1b62318696a8301eac12ca44ffd8515 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Wed, 5 Jun 2024 08:50:41 +0530 Subject: [PATCH 268/316] fix: add fix to include referred entities --- .../store/graph/v2/AtlasEntityStoreV2.java | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 846510c9ea4..7df859eb995 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -116,6 +116,7 @@ import static org.apache.atlas.repository.graph.GraphHelper.getStatus; import static org.apache.atlas.repository.store.graph.v2.EntityGraphMapper.validateLabels; import static org.apache.atlas.repository.store.graph.v2.tasks.MeaningsTaskFactory.*; +import static org.apache.atlas.repository.util.AccessControlUtils.REL_ATTR_POLICIES; import static org.apache.atlas.type.Constants.HAS_LINEAGE; import static org.apache.atlas.type.Constants.HAS_LINEAGE_VALID; import static org.apache.atlas.type.Constants.MEANINGS_TEXT_PROPERTY_KEY; @@ -2709,26 +2710,36 @@ private void repairMeanings(AtlasVertex assetVertex) { } @Override public void repairAlias(String guid) throws AtlasBaseException { - // Fetch entity with extInfo - AtlasEntity.AtlasEntityWithExtInfo entity = entityRetriever.toAtlasEntityWithExtInfo(guid); + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("repairAlias"); + // Fetch accesscontrolEntity with extInfo + AtlasEntity.AtlasEntityWithExtInfo accesscontrolEntity = entityRetriever.toAtlasEntityWithExtInfo(guid); - if (entity == null) { + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, new AtlasEntityHeader(accesscontrolEntity.getEntity()))); + + if (accesscontrolEntity == null) { throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guid); } - // Validate entity status - if (entity.getEntity().getStatus() != ACTIVE) { + // Validate accesscontrolEntity status + if (accesscontrolEntity.getEntity().getStatus() != ACTIVE) { throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_DELETED, guid); } - // Validate entity type - String entityType = entity.getEntity().getTypeName(); + // Validate accesscontrolEntity type + String entityType = accesscontrolEntity.getEntity().getTypeName(); if (!PERSONA_ENTITY_TYPE.equals(entityType)) { throw new AtlasBaseException(AtlasErrorCode.INVALID_OBJECT_ID, entityType); } + List policies = (List) accesscontrolEntity.getEntity().getRelationshipAttribute(REL_ATTR_POLICIES); + for (AtlasObjectId policy : policies) { + accesscontrolEntity.addReferredEntity(entityRetriever.toAtlasEntity(policy)); + } + // Rebuild alias - this.esAliasStore.rebuildAlias(entity); + this.esAliasStore.rebuildAlias(accesscontrolEntity); + + RequestContext.get().endMetricRecord(metric); } } From a0294805d2feddf40d33f286e0fad670b58c90b8 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Wed, 5 Jun 2024 09:54:27 +0530 Subject: [PATCH 269/316] nit: make all namings descriptive --- .../store/graph/AtlasEntityStore.java | 3 +-- .../store/graph/v2/AtlasEntityStoreV2.java | 2 +- .../org/apache/atlas/web/rest/EntityREST.java | 24 ++++++++++++++++--- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java index d072139aef5..912799cdd6d 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java @@ -25,7 +25,6 @@ import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; import org.apache.atlas.model.instance.AtlasEntityHeader; -import org.apache.atlas.model.instance.AtlasEntityHeaders; import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.instance.AtlasHasLineageRequests; import org.apache.atlas.model.instance.EntityMutationResponse; @@ -364,6 +363,6 @@ EntityMutationResponse deleteByUniqueAttributes(List objectIds) void repairMeaningAttributeForTerms(List termGuids) throws AtlasBaseException; - public void repairAlias(String guid) throws AtlasBaseException; + void repairAccesscontrolAlias(String guid) throws AtlasBaseException; } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 7df859eb995..037fadbc172 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -2709,7 +2709,7 @@ private void repairMeanings(AtlasVertex assetVertex) { } @Override - public void repairAlias(String guid) throws AtlasBaseException { + public void repairAccesscontrolAlias(String guid) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("repairAlias"); // Fetch accesscontrolEntity with extInfo AtlasEntity.AtlasEntityWithExtInfo accesscontrolEntity = entityRetriever.toAtlasEntityWithExtInfo(guid); diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java index 31283924a23..7669b078390 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java @@ -1932,9 +1932,27 @@ public void repairIndexByTypeName(@PathParam("typename") String typename, @Query } @POST - @Path("/repairalias/{guid}") + @Path("/repairAccesscontrolAlias/{guid}") @Timed - public void repairAlias(@PathParam("guid") String guid) throws AtlasBaseException { - entitiesStore.repairAlias(guid); + public void repairAccessControlAlias(@PathParam("guid") String guid) throws AtlasBaseException { + Servlets.validateQueryParamLength("guid", guid); + + AtlasPerfTracer perf = null; + + + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityREST.repairAccessControlAlias"); + } + + entitiesStore.repairAccesscontrolAlias(guid); + + LOG.info("Repaired access control alias for entity with guid {}", guid); + + } finally { + AtlasPerfTracer.log(perf); + } + + } } From b4a145af8d8d94cd3933d4ec782e16f475857890 Mon Sep 17 00:00:00 2001 From: arpit-at Date: Wed, 5 Jun 2024 11:29:43 +0530 Subject: [PATCH 270/316] to build for beta test --- .github/workflows/maven.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 0977cb36a25..bb5955c5489 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,6 +26,7 @@ on: - development - master - lineageondemand + - connectionadmins jobs: build: From b9b38aebaf519c07185dfb5df191556519aaad7c Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Wed, 5 Jun 2024 13:22:57 +0530 Subject: [PATCH 271/316] fix: remove unwanted funcs --- .../store/aliasstore/ESAliasStore.java | 28 ------------------- .../store/graph/v2/AtlasEntityStoreV2.java | 8 ++---- 2 files changed, 2 insertions(+), 34 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/aliasstore/ESAliasStore.java b/repository/src/main/java/org/apache/atlas/repository/store/aliasstore/ESAliasStore.java index 88fdf0f4af3..1991106b965 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/aliasstore/ESAliasStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/aliasstore/ESAliasStore.java @@ -141,23 +141,6 @@ public boolean updateAlias(AtlasEntity.AtlasEntityWithExtInfo accessControl, Atl return true; } - public void rebuildAlias(AtlasEntity.AtlasEntityWithExtInfo accessControl) throws AtlasBaseException { - String aliasName = getAliasName(accessControl.getEntity()); - - Map filter; - - if (PERSONA_ENTITY_TYPE.equals(accessControl.getEntity().getTypeName())) { - filter = getFilterForPersona(accessControl); - } else { - filter = getFilterForPurpose(accessControl.getEntity()); - } - - ESAliasRequestBuilder requestBuilder = new ESAliasRequestBuilder(); - requestBuilder.addAction(ADD, new AliasAction(getIndexNameFromAliasIfExists(VERTEX_INDEX_NAME), aliasName, filter)); - - graph.createOrUpdateESAlias(requestBuilder); - } - @Override public boolean deleteAlias(String aliasName) throws AtlasBaseException { graph.deleteESAlias(getIndexNameFromAliasIfExists(VERTEX_INDEX_NAME), aliasName); @@ -178,17 +161,6 @@ private Map getFilterForPersona(AtlasEntity.AtlasEntityWithExtIn return esClausesToFilter(allowClauseList); } - private Map getFilterForPersona(AtlasEntity.AtlasEntityWithExtInfo persona) throws AtlasBaseException { - List> allowClauseList = new ArrayList<>(); - - List policies = getPolicies(persona); - if (CollectionUtils.isNotEmpty(policies)) { - personaPolicyToESDslClauses(policies, allowClauseList); - } - - return esClausesToFilter(allowClauseList); - } - private Map getFilterForPurpose(AtlasEntity purpose) throws AtlasBaseException { List> allowClauseList = new ArrayList<>(); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 037fadbc172..83df0fe2efe 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -2716,10 +2716,6 @@ public void repairAccesscontrolAlias(String guid) throws AtlasBaseException { AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, new AtlasEntityHeader(accesscontrolEntity.getEntity()))); - if (accesscontrolEntity == null) { - throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guid); - } - // Validate accesscontrolEntity status if (accesscontrolEntity.getEntity().getStatus() != ACTIVE) { throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_DELETED, guid); @@ -2728,7 +2724,7 @@ public void repairAccesscontrolAlias(String guid) throws AtlasBaseException { // Validate accesscontrolEntity type String entityType = accesscontrolEntity.getEntity().getTypeName(); if (!PERSONA_ENTITY_TYPE.equals(entityType)) { - throw new AtlasBaseException(AtlasErrorCode.INVALID_OBJECT_ID, entityType); + throw new AtlasBaseException(AtlasErrorCode.OPERATION_NOT_SUPPORTED, entityType); } List policies = (List) accesscontrolEntity.getEntity().getRelationshipAttribute(REL_ATTR_POLICIES); @@ -2737,7 +2733,7 @@ public void repairAccesscontrolAlias(String guid) throws AtlasBaseException { } // Rebuild alias - this.esAliasStore.rebuildAlias(accesscontrolEntity); + this.esAliasStore.updateAlias(accesscontrolEntity, null); RequestContext.get().endMetricRecord(metric); } From dbf011bc321480a760f014c7560ae16d2b0cd729 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Wed, 5 Jun 2024 13:23:47 +0530 Subject: [PATCH 272/316] nit: have a good api path --- webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java index 7669b078390..3aadc06ad22 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java @@ -1932,7 +1932,7 @@ public void repairIndexByTypeName(@PathParam("typename") String typename, @Query } @POST - @Path("/repairAccesscontrolAlias/{guid}") + @Path("/repair/accesscontrolAlias/{guid}") @Timed public void repairAccessControlAlias(@PathParam("guid") String guid) throws AtlasBaseException { Servlets.validateQueryParamLength("guid", guid); From c7465f1bd49431a189350594400535c9f82b7e2e Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Wed, 5 Jun 2024 13:58:22 +0530 Subject: [PATCH 273/316] fix:return on no update to avoid deleting attribute --- .../atlas/repository/store/graph/v2/EntityGraphMapper.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index f18d802ffc2..fcecb31fd56 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -2216,6 +2216,11 @@ private void addInternalProductAttr(AttributeMutationContext ctx, List c AtlasVertex toVertex = ctx.getReferringVertex(); String toVertexType = getTypeName(toVertex); + if(currentElements.isEmpty() && createdElements.isEmpty() && deletedElements.isEmpty()){ + RequestContext.get().endMetricRecord(metricRecorder); + return; + } + if (TYPE_PRODUCT.equals(toVertexType)) { if(currentElements == null){ From 4d9ec542118e21ac69c35aba7c2e6c9326b54a61 Mon Sep 17 00:00:00 2001 From: arpit-at Date: Wed, 5 Jun 2024 14:17:37 +0530 Subject: [PATCH 274/316] DG-1374 Update in pre-processor to handle the admin attributes --- .github/workflows/maven.yml | 1 - .../store/graph/v2/preprocessor/ConnectionPreProcessor.java | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index bb5955c5489..0977cb36a25 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,7 +26,6 @@ on: - development - master - lineageondemand - - connectionadmins jobs: build: diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java index c8ff645151a..2301b5f3405 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java @@ -196,7 +196,7 @@ private void processUpdateConnection(EntityMutationContext context, AtlasStruct List finalStateRoles = emptyRole ? new ArrayList<>() : newAdminRoles; List finalStateGroups = emptyGroup ? new ArrayList<>() : newAdminGroups; - if (CollectionUtils.isNotEmpty(finalStateUsers) && CollectionUtils.isEmpty(finalStateGroups) && CollectionUtils.isEmpty(finalStateRoles)) { + if (CollectionUtils.isEmpty(finalStateUsers) && CollectionUtils.isEmpty(finalStateGroups) && CollectionUtils.isEmpty(finalStateRoles)) { throw new AtlasBaseException(AtlasErrorCode.ADMIN_LIST_SHOULD_NOT_BE_EMPTY, existingConnEntity.getTypeName()); } From 1691abd6606fddb002a2d0b2d67705b2faa5986f Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Wed, 5 Jun 2024 19:06:50 +0530 Subject: [PATCH 275/316] filter for getting active relations --- .../atlas/repository/store/graph/v2/EntityGraphMapper.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index fcecb31fd56..855b3978223 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -85,6 +85,7 @@ import org.springframework.stereotype.Component; import javax.inject.Inject; +import javax.validation.constraints.NotNull; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -2216,7 +2217,7 @@ private void addInternalProductAttr(AttributeMutationContext ctx, List c AtlasVertex toVertex = ctx.getReferringVertex(); String toVertexType = getTypeName(toVertex); - if(currentElements.isEmpty() && createdElements.isEmpty() && deletedElements.isEmpty()){ + if((currentElements.isEmpty() || currentElements == null) && createdElements.isEmpty() && deletedElements.isEmpty()){ RequestContext.get().endMetricRecord(metricRecorder); return; } @@ -2234,7 +2235,7 @@ private void addInternalProductAttr(AttributeMutationContext ctx, List c } if(ctx.getAttribute().getRelationshipEdgeLabel().equals(OUTPUT_PORT_PRODUCT_EDGE_LABEL)){ - addOrRemoveInternalAttr(toVertex, OUTPUT_PORT_GUIDS_ATTR, createdElements, currentElements, deletedElements); + addOrRemoveInternalAttr(toVertex, "OutputPortGuids", createdElements, currentElements, deletedElements); } if (ctx.getAttribute().getRelationshipEdgeLabel().equals(INPUT_PORT_PRODUCT_EDGE_LABEL)) { addOrRemoveInternalAttr(toVertex, INPUT_PORT_GUIDS_ATTR, createdElements, currentElements, deletedElements); @@ -2250,6 +2251,7 @@ private void addOrRemoveInternalAttr(AtlasVertex toVertex, String internalAttr, if (CollectionUtils.isNotEmpty(currentElements) && CollectionUtils.isEmpty(portGuids)) { List currentGuids = currentElements.stream() + .filter(x -> ((AtlasEdge) x).getProperty(STATE_PROPERTY_KEY, String.class).equals("ACTIVE")) .map(x -> ((AtlasEdge) x).getOutVertex().getProperty("__guid", String.class)) .collect(Collectors.toList()); From c5a92b949752254ab912dd629c5f2a7832de4ec3 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Wed, 5 Jun 2024 20:33:20 +0530 Subject: [PATCH 276/316] removed unused import --- .../atlas/repository/store/graph/v2/EntityGraphMapper.java | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 855b3978223..db91b27dafa 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -85,7 +85,6 @@ import org.springframework.stereotype.Component; import javax.inject.Inject; -import javax.validation.constraints.NotNull; import java.util.ArrayList; import java.util.Collection; import java.util.Collections; @@ -1908,7 +1907,7 @@ public List mapArrayValue(AttributeMutationContext ctx, EntityMutationContext co AtlasAttribute inverseRefAttribute = attribute.getInverseRefAttribute(); Cardinality cardinality = attribute.getAttributeDef().getCardinality(); List removedElements = new ArrayList<>(); - List newElementsCreated = new ArrayList<>(); + List newElementsCreated = new ArrayList<>(); List allArrayElements = null; List currentElements; boolean deleteExistingRelations = shouldDeleteExistingRelations(ctx, attribute); @@ -2235,7 +2234,7 @@ private void addInternalProductAttr(AttributeMutationContext ctx, List c } if(ctx.getAttribute().getRelationshipEdgeLabel().equals(OUTPUT_PORT_PRODUCT_EDGE_LABEL)){ - addOrRemoveInternalAttr(toVertex, "OutputPortGuids", createdElements, currentElements, deletedElements); + addOrRemoveInternalAttr(toVertex, OUTPUT_PORT_GUIDS_ATTR, createdElements, currentElements, deletedElements); } if (ctx.getAttribute().getRelationshipEdgeLabel().equals(INPUT_PORT_PRODUCT_EDGE_LABEL)) { addOrRemoveInternalAttr(toVertex, INPUT_PORT_GUIDS_ATTR, createdElements, currentElements, deletedElements); From 597f3ae7707cd47cb723c0a4468daf3f365e3e40 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Wed, 5 Jun 2024 20:36:14 +0530 Subject: [PATCH 277/316] lint change --- .../atlas/repository/store/graph/v2/EntityGraphMapper.java | 1 - 1 file changed, 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index db91b27dafa..43e6d1fe74b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -2128,7 +2128,6 @@ public List removeArrayValue(AttributeMutationContext ctx, EntityMutationContext boolean isNewElementsNull = elementsDeleted == null; - if (isNewElementsNull) { elementsDeleted = new ArrayList(); } From df1d5cfaa40a2b5b0195d3eb4dae836f6ff99d83 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Wed, 5 Jun 2024 20:55:31 +0530 Subject: [PATCH 278/316] fix:handle null for currentElements --- .../atlas/repository/store/graph/v2/EntityGraphMapper.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 43e6d1fe74b..6c915e19718 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -2039,7 +2039,6 @@ public List appendArrayValue(AttributeMutationContext ctx, EntityMutationContext boolean isNewElementsNull = newElements == null; - if (isNewElementsNull) { newElements = new ArrayList(); } @@ -2215,7 +2214,7 @@ private void addInternalProductAttr(AttributeMutationContext ctx, List c AtlasVertex toVertex = ctx.getReferringVertex(); String toVertexType = getTypeName(toVertex); - if((currentElements.isEmpty() || currentElements == null) && createdElements.isEmpty() && deletedElements.isEmpty()){ + if((currentElements == null || currentElements.isEmpty()) && createdElements.isEmpty() && deletedElements.isEmpty()){ RequestContext.get().endMetricRecord(metricRecorder); return; } From 1ccd85cb846435d14b8925f8805e490d84693e84 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Thu, 6 Jun 2024 15:28:23 +0530 Subject: [PATCH 279/316] removed logic to fetch currentElements --- .../store/graph/v2/EntityGraphMapper.java | 54 +++++-------------- 1 file changed, 13 insertions(+), 41 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 6c915e19718..96446d5b9dd 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -2008,7 +2008,7 @@ public List mapArrayValue(AttributeMutationContext ctx, EntityMutationContext co case INPUT_PORT_PRODUCT_EDGE_LABEL: case OUTPUT_PORT_PRODUCT_EDGE_LABEL: - addInternalProductAttr(ctx, newElementsCreated, removedElements, currentElements); + addInternalProductAttr(ctx, newElementsCreated, removedElements); break; } @@ -2098,7 +2098,7 @@ public List appendArrayValue(AttributeMutationContext ctx, EntityMutationContext case INPUT_PORT_PRODUCT_EDGE_LABEL: case OUTPUT_PORT_PRODUCT_EDGE_LABEL: - addInternalProductAttr(ctx, newElementsCreated, new ArrayList<>(0), null); + addInternalProductAttr(ctx, newElementsCreated, null); break; } @@ -2209,65 +2209,37 @@ private void addEdgesToContext(String guid, List newElementsCreated, Lis } } - private void addInternalProductAttr(AttributeMutationContext ctx, List createdElements, List deletedElements, List currentElements) throws AtlasBaseException { + private void addInternalProductAttr(AttributeMutationContext ctx, List createdElements, List deletedElements) throws AtlasBaseException { MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("addInternalProductAttrForAppend"); AtlasVertex toVertex = ctx.getReferringVertex(); String toVertexType = getTypeName(toVertex); - if((currentElements == null || currentElements.isEmpty()) && createdElements.isEmpty() && deletedElements.isEmpty()){ + if (CollectionUtils.isEmpty(createdElements) && CollectionUtils.isEmpty(deletedElements)){ RequestContext.get().endMetricRecord(metricRecorder); return; } if (TYPE_PRODUCT.equals(toVertexType)) { + String attrName = ctx.getAttribute().getRelationshipEdgeLabel().equals(OUTPUT_PORT_PRODUCT_EDGE_LABEL) + ? OUTPUT_PORT_GUIDS_ATTR + : INPUT_PORT_GUIDS_ATTR; - if(currentElements == null){ - AtlasAttribute attribute = ctx.getAttribute(); - AtlasArrayType arrType = (AtlasArrayType) attribute.getAttributeType(); - AtlasType elementType = arrType.getElementType(); - boolean isStructType = (TypeCategory.STRUCT == elementType.getTypeCategory()) || - (TypeCategory.STRUCT == attribute.getDefinedInType().getTypeCategory()); - - currentElements = (List) getCollectionElementsUsingRelationship(ctx.getReferringVertex(), attribute, isStructType); - } - - if(ctx.getAttribute().getRelationshipEdgeLabel().equals(OUTPUT_PORT_PRODUCT_EDGE_LABEL)){ - addOrRemoveInternalAttr(toVertex, OUTPUT_PORT_GUIDS_ATTR, createdElements, currentElements, deletedElements); - } - if (ctx.getAttribute().getRelationshipEdgeLabel().equals(INPUT_PORT_PRODUCT_EDGE_LABEL)) { - addOrRemoveInternalAttr(toVertex, INPUT_PORT_GUIDS_ATTR, createdElements, currentElements, deletedElements); - } + addOrRemoveDaapInternalAttr(toVertex, attrName, createdElements, deletedElements); }else{ throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Can not update product relations while updating any asset"); } RequestContext.get().endMetricRecord(metricRecorder); } - private void addOrRemoveInternalAttr(AtlasVertex toVertex, String internalAttr, List createdElements, List currentElements, List deletedElements) { - List portGuids = toVertex.getMultiValuedProperty(internalAttr, String.class); - - if (CollectionUtils.isNotEmpty(currentElements) && CollectionUtils.isEmpty(portGuids)) { - List currentGuids = currentElements.stream() - .filter(x -> ((AtlasEdge) x).getProperty(STATE_PROPERTY_KEY, String.class).equals("ACTIVE")) - .map(x -> ((AtlasEdge) x).getOutVertex().getProperty("__guid", String.class)) - .collect(Collectors.toList()); - - portGuids.addAll(currentGuids); - } - + private void addOrRemoveDaapInternalAttr(AtlasVertex toVertex, String internalAttr, List createdElements, List deletedElements) { if (CollectionUtils.isNotEmpty(createdElements)) { - List assetGuids = createdElements.stream().map(x -> ((AtlasEdge) x).getOutVertex().getProperty("__guid", String.class)).collect(Collectors.toList()); - portGuids = (List) CollectionUtils.union(portGuids, assetGuids); + List addedGuids = createdElements.stream().map(x -> ((AtlasEdge) x).getOutVertex().getProperty("__guid", String.class)).collect(Collectors.toList()); + addedGuids.forEach(guid -> AtlasGraphUtilsV2.addEncodedProperty(toVertex, internalAttr, guid)); } if (CollectionUtils.isNotEmpty(deletedElements)) { - List assetGuids = deletedElements.stream().map(x -> x.getOutVertex().getProperty("__guid", String.class)).collect(Collectors.toList()); - portGuids.removeAll(assetGuids); - } - - toVertex.removeProperty(internalAttr); - if (CollectionUtils.isNotEmpty(portGuids)) { - portGuids.forEach(guid -> AtlasGraphUtilsV2.addEncodedProperty(toVertex, internalAttr , guid)); + List removedGuids = deletedElements.stream().map(x -> x.getOutVertex().getProperty("__guid", String.class)).collect(Collectors.toList()); + removedGuids.forEach(guid -> AtlasGraphUtilsV2.removeItemFromListPropertyValue(toVertex, internalAttr, guid)); } } From ac780883b7ae4ad38ee2542e61d25560f615731c Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Thu, 6 Jun 2024 15:35:41 +0530 Subject: [PATCH 280/316] fix: remove flow function call --- .../atlas/repository/store/graph/v2/EntityGraphMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 96446d5b9dd..596420696d4 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -2170,7 +2170,7 @@ public List removeArrayValue(AttributeMutationContext ctx, EntityMutationContext case INPUT_PORT_PRODUCT_EDGE_LABEL: case OUTPUT_PORT_PRODUCT_EDGE_LABEL: - addInternalProductAttr(ctx, new ArrayList<>(0) , removedElements, null); + addInternalProductAttr(ctx, null , removedElements); break; } From ce853b290137b4b976eaedef7c9de6bd8c7a2011 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Thu, 6 Jun 2024 16:12:44 +0530 Subject: [PATCH 281/316] preventing update on internal attributes in pre-processor --- .../v2/preprocessor/datamesh/DataProductPreProcessor.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java index ea974535b6c..0851baa95ce 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataProductPreProcessor.java @@ -81,6 +81,9 @@ private void processCreateProduct(AtlasEntity entity,AtlasVertex vertex) throws String productName = (String) entity.getAttribute(NAME); String parentDomainQualifiedName = ""; + entity.removeAttribute(OUTPUT_PORT_GUIDS_ATTR); + entity.removeAttribute(INPUT_PORT_GUIDS_ATTR); + if (parentDomainObject == null) { throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Cannot create a Product without a Domain Relationship"); } else { @@ -109,6 +112,9 @@ private void processCreateProduct(AtlasEntity entity,AtlasVertex vertex) throws private void processUpdateProduct(AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdateProduct"); + entity.removeAttribute(OUTPUT_PORT_GUIDS_ATTR); + entity.removeAttribute(INPUT_PORT_GUIDS_ATTR); + if(entity.hasRelationshipAttribute(DATA_DOMAIN_REL_TYPE) && entity.getRelationshipAttribute(DATA_DOMAIN_REL_TYPE) == null){ throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "DataProduct can only be moved to another Domain."); } From 492809d2b479f3363449952482f473c405a82b0c Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Thu, 6 Jun 2024 16:37:34 +0530 Subject: [PATCH 282/316] blocked relationship modification in relationship APIs for input and output ports --- .../org/apache/atlas/repository/Constants.java | 4 ++++ .../store/graph/v2/AtlasRelationshipStoreV2.java | 14 +++----------- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index a1a0e1278f9..9408328b9f4 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -149,6 +149,10 @@ public final class Constants { public static final String REL_DOMAIN_TO_STAKEHOLDERS = "data_domain_stakeholders"; public static final String REL_STAKEHOLDER_TITLE_TO_STAKEHOLDERS = "stakeholder_title_stakeholders"; + + public static final String REL_DATA_PRODUCT_TO_OUTPUT_PORTS = "data_products_output_ports"; + public static final String REL_DATA_PRODUCT_TO_INPUT_PORTS = "data_products_input_ports"; + public static final String INPUT_PORT_PRODUCT_EDGE_LABEL = "__Asset.inputPortDataProducts"; public static final String OUTPUT_PORT_PRODUCT_EDGE_LABEL = "__Asset.outputPortDataProducts"; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java index 14599421920..afdf2825f14 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasRelationshipStoreV2.java @@ -68,17 +68,7 @@ import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.NONE; import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.ONE_TO_TWO; import static org.apache.atlas.model.typedef.AtlasRelationshipDef.PropagateTags.TWO_TO_ONE; -import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY; -import static org.apache.atlas.repository.Constants.HOME_ID_KEY; -import static org.apache.atlas.repository.Constants.PROVENANCE_TYPE_KEY; -import static org.apache.atlas.repository.Constants.RELATIONSHIPTYPE_TAG_PROPAGATION_KEY; -import static org.apache.atlas.repository.Constants.RELATIONSHIP_GUID_PROPERTY_KEY; -import static org.apache.atlas.repository.Constants.REL_DOMAIN_TO_DOMAINS; -import static org.apache.atlas.repository.Constants.REL_DOMAIN_TO_PRODUCTS; -import static org.apache.atlas.repository.Constants.REL_DOMAIN_TO_STAKEHOLDERS; -import static org.apache.atlas.repository.Constants.REL_POLICY_TO_ACCESS_CONTROL; -import static org.apache.atlas.repository.Constants.REL_STAKEHOLDER_TITLE_TO_STAKEHOLDERS; -import static org.apache.atlas.repository.Constants.VERSION_PROPERTY_KEY; +import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.graph.GraphHelper.getTypeName; import static org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2.*; import static org.apache.atlas.repository.store.graph.v2.tasks.ClassificationPropagateTaskFactory.CLASSIFICATION_PROPAGATION_RELATIONSHIP_UPDATE; @@ -116,6 +106,8 @@ public class AtlasRelationshipStoreV2 implements AtlasRelationshipStore { add(REL_DOMAIN_TO_STAKEHOLDERS); add(REL_STAKEHOLDER_TITLE_TO_STAKEHOLDERS); add(REL_POLICY_TO_ACCESS_CONTROL); + add(REL_DATA_PRODUCT_TO_OUTPUT_PORTS); + add(REL_DATA_PRODUCT_TO_INPUT_PORTS); }}; public enum RelationshipMutation { From 49f56a1d75ea864e6013724affa94dbb4872a449 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Thu, 6 Jun 2024 20:37:49 +0530 Subject: [PATCH 283/316] Migration Service --- .../v2/DataMeshAttrMigrationService.java | 108 ++++++++++++++++++ .../v2/preprocessor/PreProcessorUtils.java | 3 + 2 files changed, 111 insertions(+) create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshAttrMigrationService.java diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshAttrMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshAttrMigrationService.java new file mode 100644 index 00000000000..be7a44e9bc5 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshAttrMigrationService.java @@ -0,0 +1,108 @@ +package org.apache.atlas.repository.store.graph.v2; + +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.service.redis.RedisService; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.commons.collections.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; + +import static org.apache.atlas.repository.Constants.*; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.*; + +public class DataMeshAttrMigrationService implements MigrationService { + + private static final Logger LOG = LoggerFactory.getLogger(DataMeshQNMigrationService.class); + + private final EntityGraphRetriever entityRetriever; + + private final AtlasTypeRegistry typeRegistry; + private final RedisService redisService; + + private String productGuid; + private boolean forceRegen; + private final TransactionInterceptHelper transactionInterceptHelper; + + public DataMeshAttrMigrationService(EntityGraphRetriever entityRetriever, String productGuid, AtlasTypeRegistry typeRegistry, TransactionInterceptHelper transactionInterceptHelper, RedisService redisService, boolean forceRegen) { + this.entityRetriever = entityRetriever; + this.typeRegistry = typeRegistry; + this.redisService = redisService; + this.transactionInterceptHelper = transactionInterceptHelper; + this.forceRegen = forceRegen; + this.productGuid = productGuid; + } + + public void startMigration() throws Exception { + try { + redisService.putValue(DATA_MESH_ATTR, MigrationStatus.IN_PROGRESS.name()); + + AtlasVertex productVertex = entityRetriever.getEntityVertex(this.productGuid); + migrateAttr(productVertex); + commitChanges(); + } catch (Exception e) { + LOG.error("Migration failed", e); + redisService.putValue(DATA_MESH_ATTR, MigrationStatus.FAILED.name()); + throw e; + } + + redisService.putValue(DATA_MESH_ATTR, MigrationStatus.SUCCESSFUL.name()); + } + + private void migrateAttr(AtlasVertex vertex) throws AtlasBaseException { + List outputPorts = vertex.getMultiValuedProperty("outputPorts", Object.class); + List outputPortGuids = vertex.getMultiValuedProperty(OUTPUT_PORT_GUIDS_ATTR, String.class); + + List inputPorts = vertex.getMultiValuedProperty("inputPorts", Object.class); + List inputPortGuids = vertex.getMultiValuedProperty(INPUT_PORT_GUIDS_ATTR, String.class); + + if(outputPorts.size() != outputPortGuids.size()) { + LOG.info("Migrating outputPort guid attribute: {} for Product: {}", OUTPUT_PORT_GUIDS_ATTR, vertex.getProperty(QUALIFIED_NAME, String.class)); + addGuids(vertex, OUTPUT_PORT_GUIDS_ATTR, outputPorts, outputPortGuids); + } + + if(inputPorts.size() != inputPortGuids.size()) { + LOG.info("Migrating inputPort guid attribute: {} for Product: {}", INPUT_PORT_GUIDS_ATTR, vertex.getProperty(QUALIFIED_NAME, String.class)); + addGuids(vertex, INPUT_PORT_GUIDS_ATTR, inputPorts, inputPortGuids); + } + } + + public void commitChanges() throws AtlasBaseException { + try { + transactionInterceptHelper.intercept(); + LOG.info("Committed a entity to the graph"); + } catch (Exception e){ + LOG.error("Failed to commit asset: ", e); + throw e; + } + } + + private void addGuids(AtlasVertex productVertex, String internalAttr, List currentElements, List currentGuids){ + currentGuids = new ArrayList<>(); + for(Object port : currentElements){ + if(port instanceof Map){ + Map portMap = (Map) port; + currentGuids.add((String) portMap.get("guid")); + } + } + + if (CollectionUtils.isNotEmpty(currentGuids)) { + currentGuids.forEach(guid -> AtlasGraphUtilsV2.addEncodedProperty(productVertex, internalAttr , guid)); + } + } + + @Override + public void run() { + try { + LOG.info("Starting migration: {}", DATA_MESH_ATTR); + startMigration(); + LOG.info("Finished migration: {}", DATA_MESH_ATTR); + } catch (Exception e) { + LOG.error("Error running migration : {}",e.toString()); + throw new RuntimeException(e); + } + } +} \ No newline at end of file diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index 2680e482079..3b7c1f5f90a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -58,10 +58,13 @@ public class PreProcessorUtils { public static final String DAAP_VISIBILITY_ATTR = "daapVisibility"; public static final String DAAP_VISIBILITY_USERS_ATTR = "daapVisibilityUsers"; public static final String DAAP_VISIBILITY_GROUPS_ATTR = "daapVisibilityGroups"; + public static final String OUTPUT_PORT_GUIDS_ATTR = "daapOutputPortGuids"; + public static final String INPUT_PORT_GUIDS_ATTR = "daapInputPortGuids"; //Migration Constants public static final String MIGRATION_TYPE_PREFIX = "MIGRATION:"; public static final String DATA_MESH_QN = MIGRATION_TYPE_PREFIX + "DATA_MESH_QN"; + public static final String DATA_MESH_ATTR = MIGRATION_TYPE_PREFIX + "DATA_MESH_ATTR"; public enum MigrationStatus { IN_PROGRESS, From d82a4a083105020d78eb0f961d832aaf02cb1e3f Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Thu, 6 Jun 2024 21:43:16 +0530 Subject: [PATCH 284/316] migration service call --- .../v2/DataMeshAttrMigrationService.java | 58 ++++++++----------- .../v2/preprocessor/PreProcessorUtils.java | 2 + .../apache/atlas/web/rest/MigrationREST.java | 26 ++++++++- 3 files changed, 51 insertions(+), 35 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshAttrMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshAttrMigrationService.java index be7a44e9bc5..f6248b7f30a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshAttrMigrationService.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshAttrMigrationService.java @@ -1,7 +1,7 @@ package org.apache.atlas.repository.store.graph.v2; -import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.service.redis.RedisService; import org.apache.atlas.type.AtlasTypeRegistry; @@ -14,7 +14,7 @@ import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.*; -public class DataMeshAttrMigrationService implements MigrationService { +public class DataMeshAttrMigrationService { private static final Logger LOG = LoggerFactory.getLogger(DataMeshQNMigrationService.class); @@ -24,19 +24,17 @@ public class DataMeshAttrMigrationService implements MigrationService { private final RedisService redisService; private String productGuid; - private boolean forceRegen; private final TransactionInterceptHelper transactionInterceptHelper; - public DataMeshAttrMigrationService(EntityGraphRetriever entityRetriever, String productGuid, AtlasTypeRegistry typeRegistry, TransactionInterceptHelper transactionInterceptHelper, RedisService redisService, boolean forceRegen) { + public DataMeshAttrMigrationService(EntityGraphRetriever entityRetriever, String productGuid, AtlasTypeRegistry typeRegistry, TransactionInterceptHelper transactionInterceptHelper, RedisService redisService) { this.entityRetriever = entityRetriever; this.typeRegistry = typeRegistry; this.redisService = redisService; this.transactionInterceptHelper = transactionInterceptHelper; - this.forceRegen = forceRegen; this.productGuid = productGuid; } - public void startMigration() throws Exception { + public void migrateProduct() throws Exception { try { redisService.putValue(DATA_MESH_ATTR, MigrationStatus.IN_PROGRESS.name()); @@ -44,7 +42,7 @@ public void startMigration() throws Exception { migrateAttr(productVertex); commitChanges(); } catch (Exception e) { - LOG.error("Migration failed", e); + LOG.error("Migration failed for entity", e); redisService.putValue(DATA_MESH_ATTR, MigrationStatus.FAILED.name()); throw e; } @@ -53,20 +51,23 @@ public void startMigration() throws Exception { } private void migrateAttr(AtlasVertex vertex) throws AtlasBaseException { - List outputPorts = vertex.getMultiValuedProperty("outputPorts", Object.class); - List outputPortGuids = vertex.getMultiValuedProperty(OUTPUT_PORT_GUIDS_ATTR, String.class); + AtlasEntity productEntity = entityRetriever.toAtlasEntity(vertex); + List outputPorts = (List) productEntity.getRelationshipAttribute(OUTPUT_PORT_ATTR); + List outputPortGuids = getAssetGuids(outputPorts); + List outputPortGuidsAttr = vertex.getMultiValuedProperty(OUTPUT_PORT_GUIDS_ATTR, String.class); - List inputPorts = vertex.getMultiValuedProperty("inputPorts", Object.class); - List inputPortGuids = vertex.getMultiValuedProperty(INPUT_PORT_GUIDS_ATTR, String.class); + List inputPorts = (List) productEntity.getRelationshipAttribute(INPUT_PORT_ATTR); + List inputPortGuids = getAssetGuids(inputPorts); + List inputPortGuidsAttr = vertex.getMultiValuedProperty(INPUT_PORT_GUIDS_ATTR, String.class); - if(outputPorts.size() != outputPortGuids.size()) { + if(CollectionUtils.isEqualCollection(outputPortGuids, outputPortGuidsAttr)) { LOG.info("Migrating outputPort guid attribute: {} for Product: {}", OUTPUT_PORT_GUIDS_ATTR, vertex.getProperty(QUALIFIED_NAME, String.class)); - addGuids(vertex, OUTPUT_PORT_GUIDS_ATTR, outputPorts, outputPortGuids); + addInternalAttr(vertex, OUTPUT_PORT_GUIDS_ATTR, outputPorts, outputPortGuids); } - if(inputPorts.size() != inputPortGuids.size()) { + if(CollectionUtils.isEqualCollection(inputPortGuids, inputPortGuidsAttr)) { LOG.info("Migrating inputPort guid attribute: {} for Product: {}", INPUT_PORT_GUIDS_ATTR, vertex.getProperty(QUALIFIED_NAME, String.class)); - addGuids(vertex, INPUT_PORT_GUIDS_ATTR, inputPorts, inputPortGuids); + addInternalAttr(vertex, INPUT_PORT_GUIDS_ATTR, inputPorts, inputPortGuids); } } @@ -80,29 +81,20 @@ public void commitChanges() throws AtlasBaseException { } } - private void addGuids(AtlasVertex productVertex, String internalAttr, List currentElements, List currentGuids){ - currentGuids = new ArrayList<>(); - for(Object port : currentElements){ - if(port instanceof Map){ - Map portMap = (Map) port; - currentGuids.add((String) portMap.get("guid")); + private List getAssetGuids(List elements){ + List guids = new ArrayList<>(); + for(Object element : elements){ + if(element instanceof Map){ + Map elementMap = (Map) element; + guids.add((String) elementMap.get("guid")); } } + return guids; + } + private void addInternalAttr(AtlasVertex productVertex, String internalAttr, List currentElements, List currentGuids){ if (CollectionUtils.isNotEmpty(currentGuids)) { currentGuids.forEach(guid -> AtlasGraphUtilsV2.addEncodedProperty(productVertex, internalAttr , guid)); } } - - @Override - public void run() { - try { - LOG.info("Starting migration: {}", DATA_MESH_ATTR); - startMigration(); - LOG.info("Finished migration: {}", DATA_MESH_ATTR); - } catch (Exception e) { - LOG.error("Error running migration : {}",e.toString()); - throw new RuntimeException(e); - } - } } \ No newline at end of file diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index 3b7c1f5f90a..1f0bc8943a3 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -60,6 +60,8 @@ public class PreProcessorUtils { public static final String DAAP_VISIBILITY_GROUPS_ATTR = "daapVisibilityGroups"; public static final String OUTPUT_PORT_GUIDS_ATTR = "daapOutputPortGuids"; public static final String INPUT_PORT_GUIDS_ATTR = "daapInputPortGuids"; + public static final String OUTPUT_PORT_ATTR = "outputPorts"; + public static final String INPUT_PORT_ATTR = "inputPorts"; //Migration Constants public static final String MIGRATION_TYPE_PREFIX = "MIGRATION:"; diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java index 929c2ae3090..ea3d0918b16 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java @@ -38,8 +38,7 @@ import static org.apache.atlas.auth.client.keycloak.AtlasKeycloakClient.getKeycloakClient; import static org.apache.atlas.repository.Constants.*; -import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.DATA_MESH_QN; -import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.MIGRATION_TYPE_PREFIX; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.*; @Path("migration") @Singleton @@ -147,6 +146,29 @@ public String getMigrationStatus(@QueryParam("migrationType") String migrationTy } } + @POST + @Path("migrateProductInternalAttr") + @Timed + public Boolean migrateProductInternalAttr (@QueryParam("guid") String guid) throws Exception { + AtlasPerfTracer perf = null; + + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "MigrationREST.migrateProductInternalAttr(" + DATA_MESH_ATTR + ")"); + } + + DataMeshAttrMigrationService migrationService = new DataMeshAttrMigrationService(entityRetriever, guid, typeRegistry, transactionInterceptHelper, redisService); + migrationService.migrateProduct(); + + } catch (Exception e) { + LOG.error("Error while submitting migration", e); + return Boolean.FALSE; + } finally { + AtlasPerfTracer.log(perf); + } + return Boolean.TRUE; + } + @POST @Path("bootstrap/connections") @Timed From 03f40edafe03d95a75c393bafa3429b91c647a3f Mon Sep 17 00:00:00 2001 From: arpit-at Date: Fri, 7 Jun 2024 07:13:11 +0530 Subject: [PATCH 285/316] DG-1374 Update in pre-processor to handle the admin attributes --- .github/workflows/maven.yml | 1 + .../preprocessor/ConnectionPreProcessor.java | 62 ++++++++++++------- 2 files changed, 39 insertions(+), 24 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 0977cb36a25..bb5955c5489 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,6 +26,7 @@ on: - development - master - lineageondemand + - connectionadmins jobs: build: diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java index 2301b5f3405..12836c007a5 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -191,26 +191,45 @@ private void processUpdateConnection(EntityMutationContext context, AtlasStruct throw new AtlasBaseException(AtlasErrorCode.ADMIN_LIST_SHOULD_NOT_BE_EMPTY, existingConnEntity.getTypeName()); } - //calculate final state for all admin attributes if all 3 are empty at the end throw exception now - List finalStateUsers = emptyName ? new ArrayList<>() : newAdminUsers; - List finalStateRoles = emptyRole ? new ArrayList<>() : newAdminRoles; - List finalStateGroups = emptyGroup ? new ArrayList<>() : newAdminGroups; - - if (CollectionUtils.isEmpty(finalStateUsers) && CollectionUtils.isEmpty(finalStateGroups) && CollectionUtils.isEmpty(finalStateRoles)) { - throw new AtlasBaseException(AtlasErrorCode.ADMIN_LIST_SHOULD_NOT_BE_EMPTY, existingConnEntity.getTypeName()); - } // Update Keycloak roles RoleRepresentation representation = getKeycloakClient().getRoleByName(roleName); - updateKeycloakRoleUsers(roleName, currentAdminUsers, newAdminUsers != null ? newAdminUsers : new ArrayList<>(), representation); - updateKeycloakRoleGroups(roleName, currentAdminGroups, newAdminGroups != null ? newAdminGroups : new ArrayList<>(), representation); - updateKeycloakRoleRoles(roleName, currentAdminRoles, newAdminRoles != null ? newAdminRoles : new ArrayList<>(), representation); - + if (newAdminUsers != null) { + List finalStateUsers = determineFinalState(newAdminUsers, currentAdminUsers); + keycloakStore.updateRoleUsers(roleName, currentAdminUsers, finalStateUsers, representation); + } + if (newAdminGroups != null) { + List finalStateGroups = determineFinalState(newAdminGroups, currentAdminGroups); + keycloakStore.updateRoleGroups(roleName, currentAdminGroups, finalStateGroups, representation); + } + if (newAdminRoles != null) { + List finalStateRoles = determineFinalState(newAdminRoles, currentAdminRoles); + keycloakStore.updateRoleRoles(roleName, currentAdminRoles, finalStateRoles, representation); + } RequestContext.get().endMetricRecord(metricRecorder); } } + // if the list is empty -> we want to remove all elements + // if the list is non-empty -> we want to replace + // if the list is equal to prev value -> no update is required + private static List determineFinalState(List newAdmins, List currentAdmins) { + if (newAdmins == null || newAdmins.isEmpty()) { + return new ArrayList<>(); + } + + List sortedNewAdmins = newAdmins.stream().sorted().collect(Collectors.toList()); + List sortedCurrentAdmins = currentAdmins.stream().sorted().collect(Collectors.toList()); + + if (sortedNewAdmins.equals(sortedCurrentAdmins)) { + return new ArrayList<>(); + } + + return newAdmins; + } + + private Optional> getAttributeList(AtlasEntity entity, String attributeName) { if (entity.hasAttribute(attributeName)) { return Optional.of((List) entity.getAttribute(attributeName)); @@ -218,10 +237,6 @@ private Optional> getAttributeList(AtlasEntity entity, String attri return Optional.empty(); } - // Check if all lists are empty - private boolean areAllListEmpty(List... lists) { - return Arrays.stream(lists).allMatch(Collection::isEmpty); - } private void updateKeycloakRoleUsers(String roleName, List currentUsers, List newUsers, RoleRepresentation representation) throws AtlasBaseException { if (!newUsers.isEmpty() || !currentUsers.isEmpty()) { @@ -230,9 +245,9 @@ private void updateKeycloakRoleUsers(String roleName, List currentUsers, } private void updateKeycloakRoleGroups(String roleName, List currentGroups, List newGroups, RoleRepresentation representation) throws AtlasBaseException { - if (!newGroups.isEmpty() || !currentGroups.isEmpty()) { - keycloakStore.updateRoleGroups(roleName, currentGroups, newGroups, representation); - } + + keycloakStore.updateRoleGroups(roleName, currentGroups, newGroups, representation); + } private void updateKeycloakRoleRoles(String roleName, List currentRoles, List newRoles, RoleRepresentation representation) throws AtlasBaseException { @@ -242,7 +257,6 @@ private void updateKeycloakRoleRoles(String roleName, List currentRoles, } - @Override public void processDelete(AtlasVertex vertex) throws AtlasBaseException { // Process Delete connection role and policies in case of hard delete or purge @@ -277,7 +291,7 @@ private boolean isDeleteTypeSoft() { private List getConnectionPolicies(String guid, String roleName) throws AtlasBaseException { List ret = new ArrayList<>(); - + IndexSearchParams indexSearchParams = new IndexSearchParams(); Map dsl = new HashMap<>(); From ac7260a7e7c4258fa3036ba969ca562bbc6e9598 Mon Sep 17 00:00:00 2001 From: arpit-at Date: Fri, 7 Jun 2024 07:33:00 +0530 Subject: [PATCH 286/316] DG-1374 Update in pre-processor to handle the admin attributes --- .../v2/preprocessor/ConnectionPreProcessor.java | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java index 12836c007a5..1fc2a212be7 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java @@ -196,15 +196,21 @@ private void processUpdateConnection(EntityMutationContext context, AtlasStruct RoleRepresentation representation = getKeycloakClient().getRoleByName(roleName); if (newAdminUsers != null) { List finalStateUsers = determineFinalState(newAdminUsers, currentAdminUsers); - keycloakStore.updateRoleUsers(roleName, currentAdminUsers, finalStateUsers, representation); + if(CollectionUtils.isNotEmpty(finalStateUsers)) { + keycloakStore.updateRoleUsers(roleName, currentAdminUsers, finalStateUsers, representation); + } } if (newAdminGroups != null) { List finalStateGroups = determineFinalState(newAdminGroups, currentAdminGroups); - keycloakStore.updateRoleGroups(roleName, currentAdminGroups, finalStateGroups, representation); + if(CollectionUtils.isNotEmpty(finalStateGroups)) { + keycloakStore.updateRoleGroups(roleName, currentAdminGroups, finalStateGroups, representation); + } } if (newAdminRoles != null) { List finalStateRoles = determineFinalState(newAdminRoles, currentAdminRoles); - keycloakStore.updateRoleRoles(roleName, currentAdminRoles, finalStateRoles, representation); + if(CollectionUtils.isNotEmpty(finalStateRoles)) { + keycloakStore.updateRoleRoles(roleName, currentAdminRoles, finalStateRoles, representation); + } } RequestContext.get().endMetricRecord(metricRecorder); } From fd29555332e32974ae99998a7802c55c298872f7 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Fri, 7 Jun 2024 08:00:30 +0530 Subject: [PATCH 287/316] condition for migration corrected --- .../store/graph/v2/DataMeshAttrMigrationService.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshAttrMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshAttrMigrationService.java index f6248b7f30a..9d8464f797e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshAttrMigrationService.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshAttrMigrationService.java @@ -60,12 +60,12 @@ private void migrateAttr(AtlasVertex vertex) throws AtlasBaseException { List inputPortGuids = getAssetGuids(inputPorts); List inputPortGuidsAttr = vertex.getMultiValuedProperty(INPUT_PORT_GUIDS_ATTR, String.class); - if(CollectionUtils.isEqualCollection(outputPortGuids, outputPortGuidsAttr)) { + if(!CollectionUtils.isEqualCollection(outputPortGuids, outputPortGuidsAttr)) { LOG.info("Migrating outputPort guid attribute: {} for Product: {}", OUTPUT_PORT_GUIDS_ATTR, vertex.getProperty(QUALIFIED_NAME, String.class)); addInternalAttr(vertex, OUTPUT_PORT_GUIDS_ATTR, outputPorts, outputPortGuids); } - if(CollectionUtils.isEqualCollection(inputPortGuids, inputPortGuidsAttr)) { + if(!CollectionUtils.isEqualCollection(inputPortGuids, inputPortGuidsAttr)) { LOG.info("Migrating inputPort guid attribute: {} for Product: {}", INPUT_PORT_GUIDS_ATTR, vertex.getProperty(QUALIFIED_NAME, String.class)); addInternalAttr(vertex, INPUT_PORT_GUIDS_ATTR, inputPorts, inputPortGuids); } From fbf19a8a654103561bc241361608d980a3eff8d5 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Fri, 7 Jun 2024 11:43:58 +0530 Subject: [PATCH 288/316] Type of outputPort Object corrected --- .../store/graph/v2/DataMeshAttrMigrationService.java | 12 ++++-------- .../org/apache/atlas/web/rest/MigrationREST.java | 2 +- 2 files changed, 5 insertions(+), 9 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshAttrMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshAttrMigrationService.java index 9d8464f797e..521eaf9640c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshAttrMigrationService.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshAttrMigrationService.java @@ -2,9 +2,9 @@ import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasRelatedObjectId; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.service.redis.RedisService; -import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.commons.collections.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,15 +20,13 @@ public class DataMeshAttrMigrationService { private final EntityGraphRetriever entityRetriever; - private final AtlasTypeRegistry typeRegistry; private final RedisService redisService; private String productGuid; private final TransactionInterceptHelper transactionInterceptHelper; - public DataMeshAttrMigrationService(EntityGraphRetriever entityRetriever, String productGuid, AtlasTypeRegistry typeRegistry, TransactionInterceptHelper transactionInterceptHelper, RedisService redisService) { + public DataMeshAttrMigrationService(EntityGraphRetriever entityRetriever, String productGuid, TransactionInterceptHelper transactionInterceptHelper, RedisService redisService) { this.entityRetriever = entityRetriever; - this.typeRegistry = typeRegistry; this.redisService = redisService; this.transactionInterceptHelper = transactionInterceptHelper; this.productGuid = productGuid; @@ -84,10 +82,8 @@ public void commitChanges() throws AtlasBaseException { private List getAssetGuids(List elements){ List guids = new ArrayList<>(); for(Object element : elements){ - if(element instanceof Map){ - Map elementMap = (Map) element; - guids.add((String) elementMap.get("guid")); - } + AtlasRelatedObjectId relatedObjectId = (AtlasRelatedObjectId) element; + guids.add(relatedObjectId.getGuid()); } return guids; } diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java index ea3d0918b16..e61a639237d 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java @@ -157,7 +157,7 @@ public Boolean migrateProductInternalAttr (@QueryParam("guid") String guid) thro perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "MigrationREST.migrateProductInternalAttr(" + DATA_MESH_ATTR + ")"); } - DataMeshAttrMigrationService migrationService = new DataMeshAttrMigrationService(entityRetriever, guid, typeRegistry, transactionInterceptHelper, redisService); + DataMeshAttrMigrationService migrationService = new DataMeshAttrMigrationService(entityRetriever, guid, transactionInterceptHelper, redisService); migrationService.migrateProduct(); } catch (Exception e) { From 31381dfe598b9b87863cb6d8414cb5d60c357bfa Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Fri, 7 Jun 2024 11:59:05 +0530 Subject: [PATCH 289/316] removed redisService --- .../store/graph/v2/DataMeshAttrMigrationService.java | 10 +--------- .../java/org/apache/atlas/web/rest/MigrationREST.java | 2 +- 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshAttrMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshAttrMigrationService.java index 521eaf9640c..3fcd29ea1fc 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshAttrMigrationService.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshAttrMigrationService.java @@ -4,7 +4,6 @@ import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasRelatedObjectId; import org.apache.atlas.repository.graphdb.AtlasVertex; -import org.apache.atlas.service.redis.RedisService; import org.apache.commons.collections.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -20,32 +19,25 @@ public class DataMeshAttrMigrationService { private final EntityGraphRetriever entityRetriever; - private final RedisService redisService; private String productGuid; private final TransactionInterceptHelper transactionInterceptHelper; - public DataMeshAttrMigrationService(EntityGraphRetriever entityRetriever, String productGuid, TransactionInterceptHelper transactionInterceptHelper, RedisService redisService) { + public DataMeshAttrMigrationService(EntityGraphRetriever entityRetriever, String productGuid, TransactionInterceptHelper transactionInterceptHelper) { this.entityRetriever = entityRetriever; - this.redisService = redisService; this.transactionInterceptHelper = transactionInterceptHelper; this.productGuid = productGuid; } public void migrateProduct() throws Exception { try { - redisService.putValue(DATA_MESH_ATTR, MigrationStatus.IN_PROGRESS.name()); - AtlasVertex productVertex = entityRetriever.getEntityVertex(this.productGuid); migrateAttr(productVertex); commitChanges(); } catch (Exception e) { LOG.error("Migration failed for entity", e); - redisService.putValue(DATA_MESH_ATTR, MigrationStatus.FAILED.name()); throw e; } - - redisService.putValue(DATA_MESH_ATTR, MigrationStatus.SUCCESSFUL.name()); } private void migrateAttr(AtlasVertex vertex) throws AtlasBaseException { diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java index e61a639237d..170a9802a6f 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java @@ -157,7 +157,7 @@ public Boolean migrateProductInternalAttr (@QueryParam("guid") String guid) thro perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "MigrationREST.migrateProductInternalAttr(" + DATA_MESH_ATTR + ")"); } - DataMeshAttrMigrationService migrationService = new DataMeshAttrMigrationService(entityRetriever, guid, transactionInterceptHelper, redisService); + DataMeshAttrMigrationService migrationService = new DataMeshAttrMigrationService(entityRetriever, guid, transactionInterceptHelper); migrationService.migrateProduct(); } catch (Exception e) { From a01e96a9599b537035f15dad614a4e6691102edf Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Fri, 7 Jun 2024 12:00:53 +0530 Subject: [PATCH 290/316] removed unused argument --- .../store/graph/v2/DataMeshAttrMigrationService.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshAttrMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshAttrMigrationService.java index 3fcd29ea1fc..dba5afb716a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshAttrMigrationService.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshAttrMigrationService.java @@ -52,12 +52,12 @@ private void migrateAttr(AtlasVertex vertex) throws AtlasBaseException { if(!CollectionUtils.isEqualCollection(outputPortGuids, outputPortGuidsAttr)) { LOG.info("Migrating outputPort guid attribute: {} for Product: {}", OUTPUT_PORT_GUIDS_ATTR, vertex.getProperty(QUALIFIED_NAME, String.class)); - addInternalAttr(vertex, OUTPUT_PORT_GUIDS_ATTR, outputPorts, outputPortGuids); + addInternalAttr(vertex, OUTPUT_PORT_GUIDS_ATTR, outputPortGuids); } if(!CollectionUtils.isEqualCollection(inputPortGuids, inputPortGuidsAttr)) { LOG.info("Migrating inputPort guid attribute: {} for Product: {}", INPUT_PORT_GUIDS_ATTR, vertex.getProperty(QUALIFIED_NAME, String.class)); - addInternalAttr(vertex, INPUT_PORT_GUIDS_ATTR, inputPorts, inputPortGuids); + addInternalAttr(vertex, INPUT_PORT_GUIDS_ATTR, inputPortGuids); } } @@ -80,7 +80,7 @@ private List getAssetGuids(List elements){ return guids; } - private void addInternalAttr(AtlasVertex productVertex, String internalAttr, List currentElements, List currentGuids){ + private void addInternalAttr(AtlasVertex productVertex, String internalAttr, List currentGuids){ if (CollectionUtils.isNotEmpty(currentGuids)) { currentGuids.forEach(guid -> AtlasGraphUtilsV2.addEncodedProperty(productVertex, internalAttr , guid)); } From f8d52aac9f60f2e0aa7d50389c31ae1896993849 Mon Sep 17 00:00:00 2001 From: arpit-at Date: Fri, 7 Jun 2024 14:03:18 +0530 Subject: [PATCH 291/316] DG-1374 Update in pre-processor to handle the admin attributes --- .github/workflows/maven.yml | 1 - .../atlas/repository/store/graph/v2/AtlasEntityStoreV2.java | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index bb5955c5489..0977cb36a25 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,7 +26,6 @@ on: - development - master - lineageondemand - - connectionadmins jobs: build: diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 2d6d28c0a72..2bfc8421c38 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -1545,8 +1545,8 @@ private EntityMutationResponse createOrUpdate(EntityStream entityStream, boolean // Notify the change listeners - entityChangeNotifier.onEntitiesMutated(ret, RequestContext.get().isImportInProgress()); - atlasRelationshipStore.onRelationshipsMutated(RequestContext.get().getRelationshipMutationMap()); + // entityChangeNotifier.onEntitiesMutated(ret, RequestContext.get().isImportInProgress()); + // atlasRelationshipStore.onRelationshipsMutated(RequestContext.get().getRelationshipMutationMap()); if (LOG.isDebugEnabled()) { LOG.debug("<== createOrUpdate()"); } From 8105ff5b0d4fb30bcda7f2cf9fe84a098a51d01a Mon Sep 17 00:00:00 2001 From: arpit-at Date: Fri, 7 Jun 2024 14:05:32 +0530 Subject: [PATCH 292/316] DG-1374 Update in pre-processor to handle the admin attributes --- .../atlas/repository/store/graph/v2/AtlasEntityStoreV2.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 2bfc8421c38..2d6d28c0a72 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -1545,8 +1545,8 @@ private EntityMutationResponse createOrUpdate(EntityStream entityStream, boolean // Notify the change listeners - // entityChangeNotifier.onEntitiesMutated(ret, RequestContext.get().isImportInProgress()); - // atlasRelationshipStore.onRelationshipsMutated(RequestContext.get().getRelationshipMutationMap()); + entityChangeNotifier.onEntitiesMutated(ret, RequestContext.get().isImportInProgress()); + atlasRelationshipStore.onRelationshipsMutated(RequestContext.get().getRelationshipMutationMap()); if (LOG.isDebugEnabled()) { LOG.debug("<== createOrUpdate()"); } From b049ef4c12387aa39a223b06359ef6576da456f4 Mon Sep 17 00:00:00 2001 From: arpit-at Date: Fri, 7 Jun 2024 14:09:34 +0530 Subject: [PATCH 293/316] DG-1374 Update in pre-processor to handle the admin attributes --- .../preprocessor/ConnectionPreProcessor.java | 21 ------------------- 1 file changed, 21 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java index 1fc2a212be7..9003a841c47 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java @@ -242,27 +242,6 @@ private Optional> getAttributeList(AtlasEntity entity, String attri } return Optional.empty(); } - - - private void updateKeycloakRoleUsers(String roleName, List currentUsers, List newUsers, RoleRepresentation representation) throws AtlasBaseException { - if (!newUsers.isEmpty() || !currentUsers.isEmpty()) { - keycloakStore.updateRoleUsers(roleName, currentUsers, newUsers, representation); - } - } - - private void updateKeycloakRoleGroups(String roleName, List currentGroups, List newGroups, RoleRepresentation representation) throws AtlasBaseException { - - keycloakStore.updateRoleGroups(roleName, currentGroups, newGroups, representation); - - } - - private void updateKeycloakRoleRoles(String roleName, List currentRoles, List newRoles, RoleRepresentation representation) throws AtlasBaseException { - if (!newRoles.isEmpty() || !currentRoles.isEmpty()) { - keycloakStore.updateRoleRoles(roleName, currentRoles, newRoles, representation); - } - } - - @Override public void processDelete(AtlasVertex vertex) throws AtlasBaseException { // Process Delete connection role and policies in case of hard delete or purge From f9672196219db05bb8d47e579945a0f70da57cd4 Mon Sep 17 00:00:00 2001 From: arpit-at Date: Fri, 7 Jun 2024 14:10:45 +0530 Subject: [PATCH 294/316] DG-1374 Update in pre-processor to handle the admin attributes --- .../store/graph/v2/preprocessor/ConnectionPreProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java index 9003a841c47..065667134e1 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java @@ -220,7 +220,7 @@ private void processUpdateConnection(EntityMutationContext context, AtlasStruct // if the list is empty -> we want to remove all elements // if the list is non-empty -> we want to replace // if the list is equal to prev value -> no update is required - private static List determineFinalState(List newAdmins, List currentAdmins) { + private List determineFinalState(List newAdmins, List currentAdmins) { if (newAdmins == null || newAdmins.isEmpty()) { return new ArrayList<>(); } From 8a5cb23a0ac344400ab84c97406af63d46ff3a81 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Fri, 7 Jun 2024 14:54:36 +0530 Subject: [PATCH 295/316] refactored code --- ...ProductInputsOutputsMigrationService.java} | 39 +++++++++++-------- .../v2/preprocessor/PreProcessorUtils.java | 1 - .../apache/atlas/web/rest/MigrationREST.java | 10 ++--- 3 files changed, 27 insertions(+), 23 deletions(-) rename repository/src/main/java/org/apache/atlas/repository/store/graph/v2/{DataMeshAttrMigrationService.java => DataProductInputsOutputsMigrationService.java} (61%) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshAttrMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataProductInputsOutputsMigrationService.java similarity index 61% rename from repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshAttrMigrationService.java rename to repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataProductInputsOutputsMigrationService.java index dba5afb716a..ddbd86426cc 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataMeshAttrMigrationService.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataProductInputsOutputsMigrationService.java @@ -1,5 +1,6 @@ package org.apache.atlas.repository.store.graph.v2; +import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasRelatedObjectId; @@ -10,12 +11,11 @@ import java.util.*; -import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.*; -public class DataMeshAttrMigrationService { +public class DataProductInputsOutputsMigrationService { - private static final Logger LOG = LoggerFactory.getLogger(DataMeshQNMigrationService.class); + private static final Logger LOG = LoggerFactory.getLogger(DataProductInputsOutputsMigrationService.class); private final EntityGraphRetriever entityRetriever; @@ -23,7 +23,7 @@ public class DataMeshAttrMigrationService { private String productGuid; private final TransactionInterceptHelper transactionInterceptHelper; - public DataMeshAttrMigrationService(EntityGraphRetriever entityRetriever, String productGuid, TransactionInterceptHelper transactionInterceptHelper) { + public DataProductInputsOutputsMigrationService(EntityGraphRetriever entityRetriever, String productGuid, TransactionInterceptHelper transactionInterceptHelper) { this.entityRetriever = entityRetriever; this.transactionInterceptHelper = transactionInterceptHelper; this.productGuid = productGuid; @@ -32,32 +32,36 @@ public DataMeshAttrMigrationService(EntityGraphRetriever entityRetriever, String public void migrateProduct() throws Exception { try { AtlasVertex productVertex = entityRetriever.getEntityVertex(this.productGuid); + if(productVertex == null) { + throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, this.productGuid); + } + migrateAttr(productVertex); commitChanges(); } catch (Exception e) { - LOG.error("Migration failed for entity", e); + LOG.error("Error while migration inputs/outputs for Dataproduct: {}", this.productGuid, e); throw e; } } private void migrateAttr(AtlasVertex vertex) throws AtlasBaseException { AtlasEntity productEntity = entityRetriever.toAtlasEntity(vertex); - List outputPorts = (List) productEntity.getRelationshipAttribute(OUTPUT_PORT_ATTR); - List outputPortGuids = getAssetGuids(outputPorts); - List outputPortGuidsAttr = vertex.getMultiValuedProperty(OUTPUT_PORT_GUIDS_ATTR, String.class); + List outputPortsRelation = (List) productEntity.getRelationshipAttribute(OUTPUT_PORT_ATTR); + List outputPortsRelationGuids = getAssetGuids(outputPortsRelation); + List outputPortGuidsAttr = (List) productEntity.getAttribute(OUTPUT_PORT_GUIDS_ATTR); - List inputPorts = (List) productEntity.getRelationshipAttribute(INPUT_PORT_ATTR); - List inputPortGuids = getAssetGuids(inputPorts); - List inputPortGuidsAttr = vertex.getMultiValuedProperty(INPUT_PORT_GUIDS_ATTR, String.class); + List inputPortsRelation = (List) productEntity.getRelationshipAttribute(INPUT_PORT_ATTR); + List inputPortsRelationGuids = getAssetGuids(inputPortsRelation); + List inputPortGuidsAttr = (List) productEntity.getAttribute(INPUT_PORT_GUIDS_ATTR); - if(!CollectionUtils.isEqualCollection(outputPortGuids, outputPortGuidsAttr)) { - LOG.info("Migrating outputPort guid attribute: {} for Product: {}", OUTPUT_PORT_GUIDS_ATTR, vertex.getProperty(QUALIFIED_NAME, String.class)); - addInternalAttr(vertex, OUTPUT_PORT_GUIDS_ATTR, outputPortGuids); + if(!CollectionUtils.isEqualCollection(outputPortsRelationGuids, outputPortGuidsAttr)) { + LOG.info("Migrating outputPort guid attribute: {} for Product: {}", OUTPUT_PORT_GUIDS_ATTR, productEntity.getGuid()); + addInternalAttr(vertex, OUTPUT_PORT_GUIDS_ATTR, outputPortsRelationGuids); } - if(!CollectionUtils.isEqualCollection(inputPortGuids, inputPortGuidsAttr)) { - LOG.info("Migrating inputPort guid attribute: {} for Product: {}", INPUT_PORT_GUIDS_ATTR, vertex.getProperty(QUALIFIED_NAME, String.class)); - addInternalAttr(vertex, INPUT_PORT_GUIDS_ATTR, inputPortGuids); + if(!CollectionUtils.isEqualCollection(inputPortsRelationGuids, inputPortGuidsAttr)) { + LOG.info("Migrating inputPort guid attribute: {} for Product: {}", INPUT_PORT_GUIDS_ATTR, productEntity.getGuid()); + addInternalAttr(vertex, INPUT_PORT_GUIDS_ATTR, inputPortsRelationGuids); } } @@ -81,6 +85,7 @@ private List getAssetGuids(List elements){ } private void addInternalAttr(AtlasVertex productVertex, String internalAttr, List currentGuids){ + productVertex.removeProperty(internalAttr); if (CollectionUtils.isNotEmpty(currentGuids)) { currentGuids.forEach(guid -> AtlasGraphUtilsV2.addEncodedProperty(productVertex, internalAttr , guid)); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index 1f0bc8943a3..4b7c4ae8598 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -66,7 +66,6 @@ public class PreProcessorUtils { //Migration Constants public static final String MIGRATION_TYPE_PREFIX = "MIGRATION:"; public static final String DATA_MESH_QN = MIGRATION_TYPE_PREFIX + "DATA_MESH_QN"; - public static final String DATA_MESH_ATTR = MIGRATION_TYPE_PREFIX + "DATA_MESH_ATTR"; public enum MigrationStatus { IN_PROGRESS, diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java index 170a9802a6f..551d0f4aa2d 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java @@ -147,22 +147,22 @@ public String getMigrationStatus(@QueryParam("migrationType") String migrationTy } @POST - @Path("migrateProductInternalAttr") + @Path("dataproduct/inputs-outputs") @Timed public Boolean migrateProductInternalAttr (@QueryParam("guid") String guid) throws Exception { AtlasPerfTracer perf = null; try { if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "MigrationREST.migrateProductInternalAttr(" + DATA_MESH_ATTR + ")"); + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "MigrationREST.migrateProductInternalAttr(" + guid + ")"); } - DataMeshAttrMigrationService migrationService = new DataMeshAttrMigrationService(entityRetriever, guid, transactionInterceptHelper); + DataProductInputsOutputsMigrationService migrationService = new DataProductInputsOutputsMigrationService(entityRetriever, guid, transactionInterceptHelper); migrationService.migrateProduct(); } catch (Exception e) { - LOG.error("Error while submitting migration", e); - return Boolean.FALSE; + LOG.error("Error while migration inputs/outputs for Dataproduct: {}", guid, e); + throw e; } finally { AtlasPerfTracer.log(perf); } From ef2d14ab92066cb6291a32874900475148207125 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Fri, 7 Jun 2024 15:36:30 +0530 Subject: [PATCH 296/316] Make common method to record mutatedDetails --- .../contract/ContractPreProcessor.java | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index 038558dac23..98add96a39b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -192,16 +192,7 @@ private void updateExistingVersion(EntityMutationContext context, AtlasEntity en AtlasEntityType entityType = ensureEntityType(entity.getTypeName()); context.addUpdated(entity.getGuid(), entity, entityType, vertex); - - AtlasEntityComparator entityComparator = new AtlasEntityComparator(typeRegistry, entityRetriever, context.getGuidAssignments(), true, true); - AtlasEntityComparator.AtlasEntityDiffResult diffResult = entityComparator.getDiffResult(entity, vertex, !storeDifferentialAudits); - RequestContext reqContext = RequestContext.get(); - if (diffResult.hasDifference()) { - if (storeDifferentialAudits) { - diffResult.getDiffEntity().setGuid(entity.getGuid()); - reqContext.cacheDifferentialEntity(diffResult.getDiffEntity()); - } - } + recordEntityMutatedDetails(context, entity, vertex); } @@ -293,11 +284,14 @@ private void datasetAttributeSync(EntityMutationContext context, AtlasEntity ass AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(entity.getGuid()); AtlasEntityType entityType = ensureEntityType(entity.getTypeName()); + context.addUpdated(entity.getGuid(), entity, entityType, vertex); + recordEntityMutatedDetails(context, entity, vertex); + } + + private void recordEntityMutatedDetails(EntityMutationContext context, AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { AtlasEntityComparator entityComparator = new AtlasEntityComparator(typeRegistry, entityRetriever, context.getGuidAssignments(), true, true); AtlasEntityComparator.AtlasEntityDiffResult diffResult = entityComparator.getDiffResult(entity, vertex, !storeDifferentialAudits); RequestContext reqContext = RequestContext.get(); - context.addUpdated(entity.getGuid(), entity, entityType, vertex); - if (diffResult.hasDifference()) { if (storeDifferentialAudits) { diffResult.getDiffEntity().setGuid(entity.getGuid()); From 3ac4a79ebd50f5a36d6b53ba0018ce3f859504f2 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Fri, 7 Jun 2024 18:11:45 +0530 Subject: [PATCH 297/316] logic to fetch only active relations --- .../apache/atlas/repository/Constants.java | 3 +- ...aProductInputsOutputsMigrationService.java | 30 +++++++++++-------- .../v2/preprocessor/PreProcessorUtils.java | 2 -- 3 files changed, 19 insertions(+), 16 deletions(-) diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 95a6e0c0078..67b63b69e86 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -150,7 +150,8 @@ public final class Constants { public static final String REL_DOMAIN_TO_STAKEHOLDERS = "data_domain_stakeholders"; public static final String REL_STAKEHOLDER_TITLE_TO_STAKEHOLDERS = "stakeholder_title_stakeholders"; - + public static final String INPUT_PORT_PRODUCT_EDGE_LABEL = "__Asset.inputPortDataProducts"; + public static final String OUTPUT_PORT_PRODUCT_EDGE_LABEL = "__Asset.outputPortDataProducts"; /** * SQL property keys. */ diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataProductInputsOutputsMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataProductInputsOutputsMigrationService.java index ddbd86426cc..65dd4b8e237 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataProductInputsOutputsMigrationService.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataProductInputsOutputsMigrationService.java @@ -4,13 +4,18 @@ import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasRelatedObjectId; +import org.apache.atlas.repository.graph.GraphHelper; +import org.apache.atlas.repository.graphdb.AtlasEdge; +import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.commons.collections.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.*; +import java.util.stream.Collectors; +import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.*; public class DataProductInputsOutputsMigrationService { @@ -45,22 +50,20 @@ public void migrateProduct() throws Exception { } private void migrateAttr(AtlasVertex vertex) throws AtlasBaseException { - AtlasEntity productEntity = entityRetriever.toAtlasEntity(vertex); - List outputPortsRelation = (List) productEntity.getRelationshipAttribute(OUTPUT_PORT_ATTR); - List outputPortsRelationGuids = getAssetGuids(outputPortsRelation); - List outputPortGuidsAttr = (List) productEntity.getAttribute(OUTPUT_PORT_GUIDS_ATTR); + List outputPortsRelationGuids = getAssetGuids(vertex, OUTPUT_PORT_PRODUCT_EDGE_LABEL); + List outputPortGuidsAttr = vertex.getMultiValuedProperty(OUTPUT_PORT_GUIDS_ATTR, String.class); - List inputPortsRelation = (List) productEntity.getRelationshipAttribute(INPUT_PORT_ATTR); - List inputPortsRelationGuids = getAssetGuids(inputPortsRelation); - List inputPortGuidsAttr = (List) productEntity.getAttribute(INPUT_PORT_GUIDS_ATTR); + + List inputPortsRelationGuids = getAssetGuids(vertex, INPUT_PORT_PRODUCT_EDGE_LABEL); + List inputPortGuidsAttr = vertex.getMultiValuedProperty(INPUT_PORT_GUIDS_ATTR, String.class); if(!CollectionUtils.isEqualCollection(outputPortsRelationGuids, outputPortGuidsAttr)) { - LOG.info("Migrating outputPort guid attribute: {} for Product: {}", OUTPUT_PORT_GUIDS_ATTR, productEntity.getGuid()); + LOG.info("Migrating outputPort guid attribute: {} for Product: {}", OUTPUT_PORT_GUIDS_ATTR, this.productGuid); addInternalAttr(vertex, OUTPUT_PORT_GUIDS_ATTR, outputPortsRelationGuids); } if(!CollectionUtils.isEqualCollection(inputPortsRelationGuids, inputPortGuidsAttr)) { - LOG.info("Migrating inputPort guid attribute: {} for Product: {}", INPUT_PORT_GUIDS_ATTR, productEntity.getGuid()); + LOG.info("Migrating inputPort guid attribute: {} for Product: {}", INPUT_PORT_GUIDS_ATTR, this.productGuid); addInternalAttr(vertex, INPUT_PORT_GUIDS_ATTR, inputPortsRelationGuids); } } @@ -75,11 +78,12 @@ public void commitChanges() throws AtlasBaseException { } } - private List getAssetGuids(List elements){ + private List getAssetGuids(AtlasVertex vertex, String edgeLabel) throws AtlasBaseException { List guids = new ArrayList<>(); - for(Object element : elements){ - AtlasRelatedObjectId relatedObjectId = (AtlasRelatedObjectId) element; - guids.add(relatedObjectId.getGuid()); + Iterator activeChildren = GraphHelper.getActiveParentVertices(vertex, edgeLabel); + while(activeChildren.hasNext()) { + AtlasVertex child = activeChildren.next(); + guids.add(child.getProperty(GUID_PROPERTY_KEY, String.class)); } return guids; } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index 4b7c4ae8598..3dc97fa642e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -60,8 +60,6 @@ public class PreProcessorUtils { public static final String DAAP_VISIBILITY_GROUPS_ATTR = "daapVisibilityGroups"; public static final String OUTPUT_PORT_GUIDS_ATTR = "daapOutputPortGuids"; public static final String INPUT_PORT_GUIDS_ATTR = "daapInputPortGuids"; - public static final String OUTPUT_PORT_ATTR = "outputPorts"; - public static final String INPUT_PORT_ATTR = "inputPorts"; //Migration Constants public static final String MIGRATION_TYPE_PREFIX = "MIGRATION:"; From 96d40a4a66ae7883fffb93cc4962504e424b0fc4 Mon Sep 17 00:00:00 2001 From: arpit-at Date: Fri, 7 Jun 2024 20:11:30 +0530 Subject: [PATCH 298/316] DG-1374 NPE handeled --- .../store/graph/v2/preprocessor/ConnectionPreProcessor.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java index 065667134e1..035aa5b503f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java @@ -238,6 +238,9 @@ private List determineFinalState(List newAdmins, List cu private Optional> getAttributeList(AtlasEntity entity, String attributeName) { if (entity.hasAttribute(attributeName)) { + if (Objects.isNull(entity.getAttribute(attributeName))) { + return Optional.of(new ArrayList<>()); + } return Optional.of((List) entity.getAttribute(attributeName)); } return Optional.empty(); From f42f2fd3515d4d1ce9dc1416f7a57af5cf804f04 Mon Sep 17 00:00:00 2001 From: arpit-at Date: Fri, 7 Jun 2024 21:09:33 +0530 Subject: [PATCH 299/316] DG-1374 Remove from roles --- .../v2/preprocessor/ConnectionPreProcessor.java | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java index 035aa5b503f..54ebae89516 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java @@ -196,19 +196,19 @@ private void processUpdateConnection(EntityMutationContext context, AtlasStruct RoleRepresentation representation = getKeycloakClient().getRoleByName(roleName); if (newAdminUsers != null) { List finalStateUsers = determineFinalState(newAdminUsers, currentAdminUsers); - if(CollectionUtils.isNotEmpty(finalStateUsers)) { + if (CollectionUtils.isNotEmpty(finalStateUsers)) { keycloakStore.updateRoleUsers(roleName, currentAdminUsers, finalStateUsers, representation); } } if (newAdminGroups != null) { List finalStateGroups = determineFinalState(newAdminGroups, currentAdminGroups); - if(CollectionUtils.isNotEmpty(finalStateGroups)) { + if (CollectionUtils.isNotEmpty(finalStateGroups)) { keycloakStore.updateRoleGroups(roleName, currentAdminGroups, finalStateGroups, representation); } } if (newAdminRoles != null) { List finalStateRoles = determineFinalState(newAdminRoles, currentAdminRoles); - if(CollectionUtils.isNotEmpty(finalStateRoles)) { + if (CollectionUtils.isNotEmpty(finalStateRoles)) { keycloakStore.updateRoleRoles(roleName, currentAdminRoles, finalStateRoles, representation); } } @@ -221,9 +221,12 @@ private void processUpdateConnection(EntityMutationContext context, AtlasStruct // if the list is non-empty -> we want to replace // if the list is equal to prev value -> no update is required private List determineFinalState(List newAdmins, List currentAdmins) { - if (newAdmins == null || newAdmins.isEmpty()) { + if (newAdmins == null) { return new ArrayList<>(); } + if (CollectionUtils.isEmpty(newAdmins)) { + return currentAdmins; + } List sortedNewAdmins = newAdmins.stream().sorted().collect(Collectors.toList()); List sortedCurrentAdmins = currentAdmins.stream().sorted().collect(Collectors.toList()); @@ -245,6 +248,7 @@ private Optional> getAttributeList(AtlasEntity entity, String attri } return Optional.empty(); } + @Override public void processDelete(AtlasVertex vertex) throws AtlasBaseException { // Process Delete connection role and policies in case of hard delete or purge From 11cb5483319d1117b4b7c07ca532ff7e4373974f Mon Sep 17 00:00:00 2001 From: arpit-at Date: Fri, 7 Jun 2024 21:21:36 +0530 Subject: [PATCH 300/316] DG-1374 Remove from roles --- .../store/graph/v2/preprocessor/ConnectionPreProcessor.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java index 54ebae89516..41a03c5d3ff 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java @@ -220,12 +220,13 @@ private void processUpdateConnection(EntityMutationContext context, AtlasStruct // if the list is empty -> we want to remove all elements // if the list is non-empty -> we want to replace // if the list is equal to prev value -> no update is required + // if the list is null -> we don't want to change private List determineFinalState(List newAdmins, List currentAdmins) { if (newAdmins == null) { - return new ArrayList<>(); + return currentAdmins; } if (CollectionUtils.isEmpty(newAdmins)) { - return currentAdmins; + return new ArrayList<>(); } List sortedNewAdmins = newAdmins.stream().sorted().collect(Collectors.toList()); From eeb8daa944708e40a235054cd072407e34d8eeb4 Mon Sep 17 00:00:00 2001 From: arpit-at Date: Fri, 7 Jun 2024 22:03:48 +0530 Subject: [PATCH 301/316] DG-1374 Remove from roles --- .../v2/preprocessor/ConnectionPreProcessor.java | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java index 41a03c5d3ff..3ece36f37d2 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java @@ -196,21 +196,18 @@ private void processUpdateConnection(EntityMutationContext context, AtlasStruct RoleRepresentation representation = getKeycloakClient().getRoleByName(roleName); if (newAdminUsers != null) { List finalStateUsers = determineFinalState(newAdminUsers, currentAdminUsers); - if (CollectionUtils.isNotEmpty(finalStateUsers)) { - keycloakStore.updateRoleUsers(roleName, currentAdminUsers, finalStateUsers, representation); - } + keycloakStore.updateRoleUsers(roleName, currentAdminUsers, finalStateUsers, representation); + } if (newAdminGroups != null) { List finalStateGroups = determineFinalState(newAdminGroups, currentAdminGroups); - if (CollectionUtils.isNotEmpty(finalStateGroups)) { - keycloakStore.updateRoleGroups(roleName, currentAdminGroups, finalStateGroups, representation); - } + keycloakStore.updateRoleGroups(roleName, currentAdminGroups, finalStateGroups, representation); + } if (newAdminRoles != null) { List finalStateRoles = determineFinalState(newAdminRoles, currentAdminRoles); - if (CollectionUtils.isNotEmpty(finalStateRoles)) { - keycloakStore.updateRoleRoles(roleName, currentAdminRoles, finalStateRoles, representation); - } + keycloakStore.updateRoleRoles(roleName, currentAdminRoles, finalStateRoles, representation); + } RequestContext.get().endMetricRecord(metricRecorder); } From 2d816317eded8466b340a33ebb9aab8c4cc5bfbb Mon Sep 17 00:00:00 2001 From: arpit-at Date: Fri, 7 Jun 2024 23:00:26 +0530 Subject: [PATCH 302/316] DG-1374 Remove from roles --- .../preprocessor/ConnectionPreProcessor.java | 21 ++++++++----------- 1 file changed, 9 insertions(+), 12 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java index 3ece36f37d2..d531b73a64e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java @@ -194,21 +194,18 @@ private void processUpdateConnection(EntityMutationContext context, AtlasStruct // Update Keycloak roles RoleRepresentation representation = getKeycloakClient().getRoleByName(roleName); - if (newAdminUsers != null) { - List finalStateUsers = determineFinalState(newAdminUsers, currentAdminUsers); - keycloakStore.updateRoleUsers(roleName, currentAdminUsers, finalStateUsers, representation); + List finalStateUsers = determineFinalState(newAdminUsers, currentAdminUsers); + keycloakStore.updateRoleUsers(roleName, currentAdminUsers, finalStateUsers, representation); - } - if (newAdminGroups != null) { - List finalStateGroups = determineFinalState(newAdminGroups, currentAdminGroups); - keycloakStore.updateRoleGroups(roleName, currentAdminGroups, finalStateGroups, representation); - } - if (newAdminRoles != null) { - List finalStateRoles = determineFinalState(newAdminRoles, currentAdminRoles); - keycloakStore.updateRoleRoles(roleName, currentAdminRoles, finalStateRoles, representation); + List finalStateGroups = determineFinalState(newAdminGroups, currentAdminGroups); + keycloakStore.updateRoleGroups(roleName, currentAdminGroups, finalStateGroups, representation); + + + List finalStateRoles = determineFinalState(newAdminRoles, currentAdminRoles); + keycloakStore.updateRoleRoles(roleName, currentAdminRoles, finalStateRoles, representation); + - } RequestContext.get().endMetricRecord(metricRecorder); } } From 40562db9036c3f7efa3e644c33830222ad79a0da Mon Sep 17 00:00:00 2001 From: arpit-at Date: Sat, 8 Jun 2024 07:13:16 +0530 Subject: [PATCH 303/316] DG-1374 equality check improved --- .../preprocessor/ConnectionPreProcessor.java | 23 ++++--------------- 1 file changed, 4 insertions(+), 19 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java index d531b73a64e..efe87fb47e5 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java @@ -195,7 +195,7 @@ private void processUpdateConnection(EntityMutationContext context, AtlasStruct // Update Keycloak roles RoleRepresentation representation = getKeycloakClient().getRoleByName(roleName); List finalStateUsers = determineFinalState(newAdminUsers, currentAdminUsers); - keycloakStore.updateRoleUsers(roleName, currentAdminUsers, finalStateUsers, representation); + keycloakStore.updateRoleUsers(roleName, currentAdminUsers, finalStateUsers, representation); List finalStateGroups = determineFinalState(newAdminGroups, currentAdminGroups); @@ -210,30 +210,15 @@ private void processUpdateConnection(EntityMutationContext context, AtlasStruct } } - + // if the list is null -> we don't want to change // if the list is empty -> we want to remove all elements // if the list is non-empty -> we want to replace - // if the list is equal to prev value -> no update is required - // if the list is null -> we don't want to change private List determineFinalState(List newAdmins, List currentAdmins) { - if (newAdmins == null) { - return currentAdmins; - } - if (CollectionUtils.isEmpty(newAdmins)) { - return new ArrayList<>(); - } - - List sortedNewAdmins = newAdmins.stream().sorted().collect(Collectors.toList()); - List sortedCurrentAdmins = currentAdmins.stream().sorted().collect(Collectors.toList()); - - if (sortedNewAdmins.equals(sortedCurrentAdmins)) { - return new ArrayList<>(); - } - - return newAdmins; + return newAdmins == null ? currentAdmins : newAdmins.isEmpty() ? new ArrayList<>() : newAdmins; } + private Optional> getAttributeList(AtlasEntity entity, String attributeName) { if (entity.hasAttribute(attributeName)) { if (Objects.isNull(entity.getAttribute(attributeName))) { From f24fe092c6a2e9be9c3c0a3f6b1fb954f9e21801 Mon Sep 17 00:00:00 2001 From: arpit-at Date: Sat, 8 Jun 2024 08:00:11 +0530 Subject: [PATCH 304/316] DG-1374 final list check added --- .../preprocessor/ConnectionPreProcessor.java | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java index efe87fb47e5..749c403998e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java @@ -195,14 +195,14 @@ private void processUpdateConnection(EntityMutationContext context, AtlasStruct // Update Keycloak roles RoleRepresentation representation = getKeycloakClient().getRoleByName(roleName); List finalStateUsers = determineFinalState(newAdminUsers, currentAdminUsers); - keycloakStore.updateRoleUsers(roleName, currentAdminUsers, finalStateUsers, representation); - - List finalStateGroups = determineFinalState(newAdminGroups, currentAdminGroups); - keycloakStore.updateRoleGroups(roleName, currentAdminGroups, finalStateGroups, representation); - - List finalStateRoles = determineFinalState(newAdminRoles, currentAdminRoles); + if (allEmpty(finalStateUsers, finalStateGroups, finalStateRoles)) { + throw new AtlasBaseException(AtlasErrorCode.ADMIN_LIST_SHOULD_NOT_BE_EMPTY, existingConnEntity.getTypeName()); + } + + keycloakStore.updateRoleUsers(roleName, currentAdminUsers, finalStateUsers, representation); + keycloakStore.updateRoleGroups(roleName, currentAdminGroups, finalStateGroups, representation); keycloakStore.updateRoleRoles(roleName, currentAdminRoles, finalStateRoles, representation); @@ -217,6 +217,12 @@ private List determineFinalState(List newAdmins, List cu return newAdmins == null ? currentAdmins : newAdmins.isEmpty() ? new ArrayList<>() : newAdmins; } + private boolean allEmpty(List... lists) { + if (lists == null || lists.length == 0) { + return true; + } + return Stream.of(lists).allMatch(list -> list != null && list.isEmpty()); + } private Optional> getAttributeList(AtlasEntity entity, String attributeName) { From 9c023cecd0498fe8be76a4c615861a046edd5a37 Mon Sep 17 00:00:00 2001 From: arpit-at Date: Mon, 10 Jun 2024 18:22:25 +0530 Subject: [PATCH 305/316] DG-1374 PR review added --- .../preprocessor/ConnectionPreProcessor.java | 28 ++++++------------- 1 file changed, 9 insertions(+), 19 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java index 749c403998e..9499ff67a94 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java @@ -6,9 +6,7 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - *

* http://www.apache.org/licenses/LICENSE-2.0 - *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -17,7 +15,6 @@ */ package org.apache.atlas.repository.store.graph.v2.preprocessor; -import com.google.common.collect.Sets; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.DeleteType; import org.apache.atlas.RequestContext; @@ -53,15 +50,11 @@ import java.util.stream.Collectors; import java.util.stream.Stream; +import static org.apache.atlas.auth.client.keycloak.AtlasKeycloakClient.getKeycloakClient; import static org.apache.atlas.authorize.AtlasAuthorizerFactory.ATLAS_AUTHORIZER_IMPL; import static org.apache.atlas.authorize.AtlasAuthorizerFactory.CURRENT_AUTHORIZER_IMPL; -import static org.apache.atlas.repository.Constants.ATTR_ADMIN_GROUPS; -import static org.apache.atlas.repository.Constants.ATTR_ADMIN_ROLES; -import static org.apache.atlas.repository.Constants.ATTR_ADMIN_USERS; -import static org.apache.atlas.repository.Constants.POLICY_ENTITY_TYPE; -import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; +import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; -import static org.apache.atlas.auth.client.keycloak.AtlasKeycloakClient.getKeycloakClient; public class ConnectionPreProcessor implements PreProcessor { private static final Logger LOG = LoggerFactory.getLogger(ConnectionPreProcessor.class); @@ -175,28 +168,25 @@ private void processUpdateConnection(EntityMutationContext context, AtlasStruct List currentAdminRoles = getAttributeList(existingConnEntity, ATTR_ADMIN_ROLES).orElseGet(ArrayList::new); // Check conditions and throw exceptions as necessary + + // If all new admin attributes are null, no action required as these are not meant to update in the request if (newAdminUsers == null && newAdminGroups == null && newAdminRoles == null) { - // If all new admin attributes are null, no action required as these are not meant to update in the request RequestContext.get().endMetricRecord(metricRecorder); return; } - // Check if any new admin attribute list is empty - boolean emptyName = newAdminUsers != null && newAdminUsers.isEmpty(); - boolean emptyGroup = newAdminGroups != null && newAdminGroups.isEmpty(); - boolean emptyRole = newAdminRoles != null && newAdminRoles.isEmpty(); - - // Throw exception if all new admin attributes are empty - if (emptyName && emptyGroup && emptyRole) { + // Throw exception if all new admin attributes are empty - + // it will handle the cases in which request have empty or null values for all three + if (CollectionUtils.isEmpty(newAdminUsers) && CollectionUtils.isEmpty(newAdminGroups) && CollectionUtils.isEmpty(newAdminRoles)) { throw new AtlasBaseException(AtlasErrorCode.ADMIN_LIST_SHOULD_NOT_BE_EMPTY, existingConnEntity.getTypeName()); } - // Update Keycloak roles RoleRepresentation representation = getKeycloakClient().getRoleByName(roleName); List finalStateUsers = determineFinalState(newAdminUsers, currentAdminUsers); List finalStateGroups = determineFinalState(newAdminGroups, currentAdminGroups); List finalStateRoles = determineFinalState(newAdminRoles, currentAdminRoles); + //this is the case where the final state after comparison with current and new value of all the attributes become empty if (allEmpty(finalStateUsers, finalStateGroups, finalStateRoles)) { throw new AtlasBaseException(AtlasErrorCode.ADMIN_LIST_SHOULD_NOT_BE_EMPTY, existingConnEntity.getTypeName()); } @@ -228,7 +218,7 @@ private boolean allEmpty(List... lists) { private Optional> getAttributeList(AtlasEntity entity, String attributeName) { if (entity.hasAttribute(attributeName)) { if (Objects.isNull(entity.getAttribute(attributeName))) { - return Optional.of(new ArrayList<>()); + return Optional.of(new ArrayList<>(0)); } return Optional.of((List) entity.getAttribute(attributeName)); } From d3f02d3226543dc63ea84aba45bfb9a45e44d588 Mon Sep 17 00:00:00 2001 From: arpit-at Date: Mon, 10 Jun 2024 18:31:32 +0530 Subject: [PATCH 306/316] DG-1374 PR review added --- .../store/graph/v2/preprocessor/ConnectionPreProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java index 9499ff67a94..b9e37102598 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java @@ -204,7 +204,7 @@ private void processUpdateConnection(EntityMutationContext context, AtlasStruct // if the list is empty -> we want to remove all elements // if the list is non-empty -> we want to replace private List determineFinalState(List newAdmins, List currentAdmins) { - return newAdmins == null ? currentAdmins : newAdmins.isEmpty() ? new ArrayList<>() : newAdmins; + return newAdmins == null ? currentAdmins : newAdmins; } private boolean allEmpty(List... lists) { From c9775be5e2d7337716c0052784f6bba43f448c84 Mon Sep 17 00:00:00 2001 From: arpit-at Date: Mon, 10 Jun 2024 19:12:51 +0530 Subject: [PATCH 307/316] DG-1374 PR review added --- .../graph/v2/preprocessor/ConnectionPreProcessor.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java index b9e37102598..9b0b83e8fd5 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/ConnectionPreProcessor.java @@ -175,12 +175,14 @@ private void processUpdateConnection(EntityMutationContext context, AtlasStruct return; } - // Throw exception if all new admin attributes are empty - - // it will handle the cases in which request have empty or null values for all three - if (CollectionUtils.isEmpty(newAdminUsers) && CollectionUtils.isEmpty(newAdminGroups) && CollectionUtils.isEmpty(newAdminRoles)) { + // Throw exception if all new admin attributes are empty but not null + boolean emptyName = newAdminUsers != null && newAdminUsers.isEmpty(); + boolean emptyGroup = newAdminGroups != null && newAdminGroups.isEmpty(); + boolean emptyRole = newAdminRoles != null && newAdminRoles.isEmpty(); + + if (emptyName && emptyGroup && emptyRole) { throw new AtlasBaseException(AtlasErrorCode.ADMIN_LIST_SHOULD_NOT_BE_EMPTY, existingConnEntity.getTypeName()); } - // Update Keycloak roles RoleRepresentation representation = getKeycloakClient().getRoleByName(roleName); List finalStateUsers = determineFinalState(newAdminUsers, currentAdminUsers); From 1c2c81dac727615ffe91bd1aa7b745d5fda28eb0 Mon Sep 17 00:00:00 2001 From: Hitesh Khandelwal Date: Mon, 10 Jun 2024 19:27:13 +0530 Subject: [PATCH 308/316] feature: Added support for "search_after" in elasticsearch queries --- .../repository/graphdb/AtlasIndexQuery.java | 4 ++-- .../graphdb/janus/AtlasElasticsearchQuery.java | 14 ++++++++++++++ .../graphdb/janus/AtlasJanusIndexQuery.java | 5 +++++ .../model/discovery/AtlasSearchResult.java | 18 ++++++++++++++++-- .../model/discovery/ElasticsearchMetadata.java | 13 +++++++++++++ .../discovery/EntityDiscoveryService.java | 1 + 6 files changed, 51 insertions(+), 4 deletions(-) diff --git a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java index 09fa0d6d060..d31f721b477 100644 --- a/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java +++ b/graphdb/api/src/main/java/org/apache/atlas/repository/graphdb/AtlasIndexQuery.java @@ -26,7 +26,7 @@ import java.util.List; import java.util.Map; import java.util.Set; - +import java.util.ArrayList; /** * A graph query that runs directly against a particular index. * @@ -103,7 +103,7 @@ interface Result { DirectIndexQueryResult getCollapseVertices(String key); Map> getHighLights(); - + ArrayList getSort(); } } diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java index 6d3ee807f77..9aa7cfe8bac 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasElasticsearchQuery.java @@ -544,6 +544,11 @@ public DirectIndexQueryResult getCollapseVerti public Map> getHighLights() { return new HashMap<>(); } + + @Override + public ArrayList getSort() { + return new ArrayList<>(); + } } @@ -609,6 +614,15 @@ public Map> getHighLights() { } return new HashMap<>(); } + + @Override + public ArrayList getSort() { + Object sort = this.hit.get("sort"); + if (Objects.nonNull(sort) && sort instanceof List) { + return (ArrayList) sort; + } + return new ArrayList<>(); + } } public class AsyncQueryResult { diff --git a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusIndexQuery.java b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusIndexQuery.java index d4417e8f6e3..f0959e48c4f 100644 --- a/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusIndexQuery.java +++ b/graphdb/janus/src/main/java/org/apache/atlas/repository/graphdb/janus/AtlasJanusIndexQuery.java @@ -153,5 +153,10 @@ public DirectIndexQueryResult getCollapseVerti public Map> getHighLights() { return new HashMap<>(); } + + @Override + public ArrayList getSort() { + return new ArrayList<>(); + } } } diff --git a/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java b/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java index 30d31b28a95..015e32a0b9d 100644 --- a/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java +++ b/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java @@ -37,6 +37,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.LinkedHashMap; import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; @@ -59,7 +60,7 @@ public class AtlasSearchResult implements Serializable { private Map aggregations; private Map searchScore; - private Map searchMetadata; + private LinkedHashMap searchMetadata; @@ -162,13 +163,26 @@ public Map getSearchMetadata() { public void addHighlights(String guid, Map> highlights) { if(MapUtils.isEmpty(this.searchMetadata)) { - this.searchMetadata = new HashMap<>(); + this.searchMetadata = new LinkedHashMap<>(); } ElasticsearchMetadata v = this.searchMetadata.getOrDefault(guid, new ElasticsearchMetadata()); v.addHighlights(highlights); this.searchMetadata.put(guid, v); } + public void addSort(String guid, ArrayList sort) { + if(MapUtils.isEmpty(this.searchMetadata)) { + this.searchMetadata = new LinkedHashMap<>(); + } + ElasticsearchMetadata v = this.searchMetadata.getOrDefault(guid, new ElasticsearchMetadata()); + v.addSort(sort); + if (this.searchMetadata.containsKey(guid)) { + this.searchMetadata.replace(guid, v); + } else { + this.searchMetadata.put(guid, v); + } + } + @Override public int hashCode() { return Objects.hash(queryType, searchParameters, queryText, type, classification, entities, attributes, fullTextResult, referredEntities, nextMarker); } diff --git a/intg/src/main/java/org/apache/atlas/model/discovery/ElasticsearchMetadata.java b/intg/src/main/java/org/apache/atlas/model/discovery/ElasticsearchMetadata.java index 270ea5e8d80..270b9c27b4c 100644 --- a/intg/src/main/java/org/apache/atlas/model/discovery/ElasticsearchMetadata.java +++ b/intg/src/main/java/org/apache/atlas/model/discovery/ElasticsearchMetadata.java @@ -5,10 +5,12 @@ import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.ArrayList; public class ElasticsearchMetadata { private Map> highlights; + private ArrayList sort; public Map> getHighlights() { return highlights; @@ -23,6 +25,17 @@ public void addHighlights(Map> highlights) { } } + public Object getSort() { return sort; } + + public void addSort(ArrayList sort) { + + if(!sort.isEmpty()) { + if (MapUtils.isEmpty(this.highlights)) { + this.sort = new ArrayList<>(); + } + this.sort = sort; + } + } @Override public String toString() { diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index 66e8d589091..f7a530cc412 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -1101,6 +1101,7 @@ private void prepareSearchResult(AtlasSearchResult ret, DirectIndexQueryResult i if (searchParams.isShowHighlights()) { ret.addHighlights(header.getGuid(), result.getHighLights()); + ret.addSort(header.getGuid(), result.getSort()); } ret.addEntity(header); From 44919386f1ff0405f0a95b5940cf9615f8274d4a Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Wed, 12 Jun 2024 09:20:42 +0530 Subject: [PATCH 309/316] removed unused imports --- ...aProductInputsOutputsMigrationService.java | 32 +++++++++++-------- 1 file changed, 18 insertions(+), 14 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataProductInputsOutputsMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataProductInputsOutputsMigrationService.java index 65dd4b8e237..2f33a32481d 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataProductInputsOutputsMigrationService.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/DataProductInputsOutputsMigrationService.java @@ -1,19 +1,13 @@ package org.apache.atlas.repository.store.graph.v2; -import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.instance.AtlasEntity; -import org.apache.atlas.model.instance.AtlasRelatedObjectId; import org.apache.atlas.repository.graph.GraphHelper; -import org.apache.atlas.repository.graphdb.AtlasEdge; -import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.commons.collections.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import java.util.*; -import java.util.stream.Collectors; import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.*; @@ -37,19 +31,25 @@ public DataProductInputsOutputsMigrationService(EntityGraphRetriever entityRetri public void migrateProduct() throws Exception { try { AtlasVertex productVertex = entityRetriever.getEntityVertex(this.productGuid); - if(productVertex == null) { - throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, this.productGuid); + + boolean isCommitRequired = migrateAttr(productVertex); + if (isCommitRequired){ + LOG.info("Committing changes for Product: {}", this.productGuid); + commitChanges(); + } + else { + LOG.info("No changes to commit for Product: {} as no migration needed", this.productGuid); } - migrateAttr(productVertex); - commitChanges(); } catch (Exception e) { LOG.error("Error while migration inputs/outputs for Dataproduct: {}", this.productGuid, e); throw e; } } - private void migrateAttr(AtlasVertex vertex) throws AtlasBaseException { + private boolean migrateAttr(AtlasVertex vertex) throws AtlasBaseException { + boolean isCommitRequired = false; + List outputPortsRelationGuids = getAssetGuids(vertex, OUTPUT_PORT_PRODUCT_EDGE_LABEL); List outputPortGuidsAttr = vertex.getMultiValuedProperty(OUTPUT_PORT_GUIDS_ATTR, String.class); @@ -60,12 +60,16 @@ private void migrateAttr(AtlasVertex vertex) throws AtlasBaseException { if(!CollectionUtils.isEqualCollection(outputPortsRelationGuids, outputPortGuidsAttr)) { LOG.info("Migrating outputPort guid attribute: {} for Product: {}", OUTPUT_PORT_GUIDS_ATTR, this.productGuid); addInternalAttr(vertex, OUTPUT_PORT_GUIDS_ATTR, outputPortsRelationGuids); + isCommitRequired = true; } if(!CollectionUtils.isEqualCollection(inputPortsRelationGuids, inputPortGuidsAttr)) { LOG.info("Migrating inputPort guid attribute: {} for Product: {}", INPUT_PORT_GUIDS_ATTR, this.productGuid); addInternalAttr(vertex, INPUT_PORT_GUIDS_ATTR, inputPortsRelationGuids); + isCommitRequired = true; } + + return isCommitRequired; } public void commitChanges() throws AtlasBaseException { @@ -80,9 +84,9 @@ public void commitChanges() throws AtlasBaseException { private List getAssetGuids(AtlasVertex vertex, String edgeLabel) throws AtlasBaseException { List guids = new ArrayList<>(); - Iterator activeChildren = GraphHelper.getActiveParentVertices(vertex, edgeLabel); - while(activeChildren.hasNext()) { - AtlasVertex child = activeChildren.next(); + Iterator activeParent = GraphHelper.getActiveParentVertices(vertex, edgeLabel); + while(activeParent.hasNext()) { + AtlasVertex child = activeParent.next(); guids.add(child.getProperty(GUID_PROPERTY_KEY, String.class)); } return guids; From 8e9b3eb1392e375e23117c2064f24739eda904cd Mon Sep 17 00:00:00 2001 From: Hitesh Khandelwal Date: Wed, 12 Jun 2024 12:18:53 +0530 Subject: [PATCH 310/316] feature: Added support for "search_after" in elasticsearch queries --- .../atlas/model/discovery/ElasticsearchMetadata.java | 3 +++ .../org/apache/atlas/model/discovery/SearchParams.java | 8 +++++++- .../apache/atlas/discovery/EntityDiscoveryService.java | 5 +++-- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/model/discovery/ElasticsearchMetadata.java b/intg/src/main/java/org/apache/atlas/model/discovery/ElasticsearchMetadata.java index 270b9c27b4c..66bd40f0389 100644 --- a/intg/src/main/java/org/apache/atlas/model/discovery/ElasticsearchMetadata.java +++ b/intg/src/main/java/org/apache/atlas/model/discovery/ElasticsearchMetadata.java @@ -1,5 +1,6 @@ package org.apache.atlas.model.discovery; +import com.fasterxml.jackson.annotation.JsonInclude; import org.apache.commons.collections.MapUtils; import java.util.HashMap; @@ -10,6 +11,8 @@ public class ElasticsearchMetadata { private Map> highlights; + + @JsonInclude(JsonInclude.Include.NON_NULL) private ArrayList sort; public Map> getHighlights() { diff --git a/intg/src/main/java/org/apache/atlas/model/discovery/SearchParams.java b/intg/src/main/java/org/apache/atlas/model/discovery/SearchParams.java index 93e6ea6b88b..6179f291a5e 100644 --- a/intg/src/main/java/org/apache/atlas/model/discovery/SearchParams.java +++ b/intg/src/main/java/org/apache/atlas/model/discovery/SearchParams.java @@ -26,6 +26,8 @@ public class SearchParams { Async async = new Async(); boolean showHighlights; + boolean showSearchMetadata; + public String getQuery() { return getQuery(); } @@ -154,10 +156,14 @@ public String getSearchInput() { return this.requestMetadata.getSearchInput(); } - public boolean isShowHighlights() { + public boolean getShowHighlights() { return showHighlights; } + public boolean getShowSearchMetadata() { + return showSearchMetadata; + } + static class RequestMetadata { private String searchInput; diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java index f7a530cc412..2a3390cfc99 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityDiscoveryService.java @@ -1098,10 +1098,11 @@ private void prepareSearchResult(AtlasSearchResult ret, DirectIndexQueryResult i header.setCollapse(collapse); } } - - if (searchParams.isShowHighlights()) { + if (searchParams.getShowSearchMetadata()) { ret.addHighlights(header.getGuid(), result.getHighLights()); ret.addSort(header.getGuid(), result.getSort()); + } else if (searchParams.getShowHighlights()) { + ret.addHighlights(header.getGuid(), result.getHighLights()); } ret.addEntity(header); From 8724ab29509d625a9934c0bdb39f96239e422beb Mon Sep 17 00:00:00 2001 From: Hitesh Khandelwal Date: Wed, 12 Jun 2024 12:26:27 +0530 Subject: [PATCH 311/316] fix names --- .../apache/atlas/model/discovery/AtlasSearchResult.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java b/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java index 015e32a0b9d..96bc0dc5871 100644 --- a/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java +++ b/intg/src/main/java/org/apache/atlas/model/discovery/AtlasSearchResult.java @@ -174,12 +174,12 @@ public void addSort(String guid, ArrayList sort) { if(MapUtils.isEmpty(this.searchMetadata)) { this.searchMetadata = new LinkedHashMap<>(); } - ElasticsearchMetadata v = this.searchMetadata.getOrDefault(guid, new ElasticsearchMetadata()); - v.addSort(sort); + ElasticsearchMetadata sortMetadata = this.searchMetadata.getOrDefault(guid, new ElasticsearchMetadata()); + sortMetadata.addSort(sort); if (this.searchMetadata.containsKey(guid)) { - this.searchMetadata.replace(guid, v); + this.searchMetadata.replace(guid, sortMetadata); } else { - this.searchMetadata.put(guid, v); + this.searchMetadata.put(guid, sortMetadata); } } From 3b2a6f14fd1d7fea64c29ee562a559f76cc9f4da Mon Sep 17 00:00:00 2001 From: Hitesh Khandelwal Date: Wed, 12 Jun 2024 16:03:19 +0530 Subject: [PATCH 312/316] remove unnecessary if condition --- .../atlas/model/discovery/ElasticsearchMetadata.java | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/model/discovery/ElasticsearchMetadata.java b/intg/src/main/java/org/apache/atlas/model/discovery/ElasticsearchMetadata.java index 66bd40f0389..8f6c9e9b3be 100644 --- a/intg/src/main/java/org/apache/atlas/model/discovery/ElasticsearchMetadata.java +++ b/intg/src/main/java/org/apache/atlas/model/discovery/ElasticsearchMetadata.java @@ -31,13 +31,7 @@ public void addHighlights(Map> highlights) { public Object getSort() { return sort; } public void addSort(ArrayList sort) { - - if(!sort.isEmpty()) { - if (MapUtils.isEmpty(this.highlights)) { - this.sort = new ArrayList<>(); - } - this.sort = sort; - } + this.sort = sort; } @Override From dff31911bb8c86d2caaf160b45920fe4196572c9 Mon Sep 17 00:00:00 2001 From: Hitesh Khandelwal Date: Wed, 12 Jun 2024 18:00:38 +0530 Subject: [PATCH 313/316] when sort is empty return null --- .../apache/atlas/model/discovery/ElasticsearchMetadata.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/intg/src/main/java/org/apache/atlas/model/discovery/ElasticsearchMetadata.java b/intg/src/main/java/org/apache/atlas/model/discovery/ElasticsearchMetadata.java index 8f6c9e9b3be..096ff82b83d 100644 --- a/intg/src/main/java/org/apache/atlas/model/discovery/ElasticsearchMetadata.java +++ b/intg/src/main/java/org/apache/atlas/model/discovery/ElasticsearchMetadata.java @@ -31,7 +31,11 @@ public void addHighlights(Map> highlights) { public Object getSort() { return sort; } public void addSort(ArrayList sort) { - this.sort = sort; + if (sort.isEmpty()) { + this.sort = null; + } else { + this.sort = sort; + } } @Override From 3ec9024fb092c8a98015e2da25537f2795f4c97c Mon Sep 17 00:00:00 2001 From: arpit-at Date: Thu, 13 Jun 2024 07:15:02 +0530 Subject: [PATCH 314/316] DG-1432 Auth Policies for MQ related Items --- .../policies/bootstrap_entity_policies.json | 184 ++++++++++++++++++ 1 file changed, 184 insertions(+) diff --git a/addons/policies/bootstrap_entity_policies.json b/addons/policies/bootstrap_entity_policies.json index 02260d558cd..23a77f5298e 100644 --- a/addons/policies/bootstrap_entity_policies.json +++ b/addons/policies/bootstrap_entity_policies.json @@ -3108,6 +3108,190 @@ "entity-read" ] } + }, + + { + "typeName": "AuthPolicy", + "attributes": { + "name": "CUD_BUSINESS_POLICY", + "qualifiedName": "CUD_BUSINESS_POLICY", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 0, + "policyUsers": [], + "policyGroups": [], + "policyRoles": [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": [ + "entity-type:BusinessPolicy", + "entity-type:BusinessPolicyException", + "entity-classification:*", + "entity:*" + ], + "policyActions": [ + "entity-create", + "entity-update", + "entity-delete" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": { + "name": "READ_BUSINESS_POLICY", + "qualifiedName": "READ_BUSINESS_POLICY", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 0, + "policyUsers": [], + "policyGroups": [], + "policyRoles": [ + "$admin", + "$guest", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": [ + "entity-type:BusinessPolicy", + "entity-type:BusinessPolicyException", + "entity-classification:*", + "entity:*" + ], + "policyActions": [ + "entity-read" + ] + } + }, + + { + "typeName": "AuthPolicy", + "attributes": { + "name": "CUD_INCIDENT", + "qualifiedName": "CUD_INCIDENT", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 0, + "policyUsers": [], + "policyGroups": [], + "policyRoles": [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": [ + "entity-type:Incident", + "entity-type:BusinessPolicyIncident", + "entity-classification:*", + "entity:*" + ], + "policyActions": [ + "entity-create", + "entity-update", + "entity-delete" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": { + "name": "READ_INCIDENT" , + "qualifiedName": "READ_INCIDENT", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 0, + "policyUsers": [], + "policyGroups": [], + "policyRoles": [ + "$admin", + "$guest", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": [ + "entity-type:Incident", + "entity-type:BusinessPolicyIncident", + "entity-classification:*", + "entity:*" + ], + "policyActions": [ + "entity-read" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": { + "name": "CRUD_BUSINESS_POLICY_LOG", + "qualifiedName": "CRUD_BUSINESS_POLICY_LOG", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 0, + "policyUsers": [], + "policyGroups": [], + "policyRoles": [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": [ + "entity-type:BusinessPolicyLog", + "entity-classification:*", + "entity:*" + ], + "policyActions": [ + "entity-create", + "entity-read", + "entity-update", + "entity-delete" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": { + "name": "CRUD_TASK", + "qualifiedName": "CRUD_TASK", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 0, + "policyUsers": [], + "policyGroups": [], + "policyRoles": [ + "$admin", + "$guest", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": [ + "entity-type:Task", + "entity-classification:*", + "entity:*" + ], + "policyActions": [ + "entity-create", + "entity-read", + "entity-update", + "entity-delete" + ] + } } ] } From 31c7a39f152987edf161c5a77a3820c421ce1441 Mon Sep 17 00:00:00 2001 From: arpit-at Date: Thu, 13 Jun 2024 11:14:35 +0530 Subject: [PATCH 315/316] DG-1432 Bootstrap relationship policies for MQ related Items --- .../bootstrap_relationship_policies.json | 76 +++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/addons/policies/bootstrap_relationship_policies.json b/addons/policies/bootstrap_relationship_policies.json index 6c44567b872..5ac53ccef3c 100644 --- a/addons/policies/bootstrap_relationship_policies.json +++ b/addons/policies/bootstrap_relationship_policies.json @@ -760,6 +760,82 @@ "remove-relationship" ] } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "LINK_BUSINESS_POLICY_BUSINESS_POLICY", + "qualifiedName": "LINK_BUSINESS_POLICY_BUSINESS_POLICY", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "RELATIONSHIP", + "policyResources": + [ + "end-one-entity-classification:*", + "end-two-entity-classification:*", + "end-one-entity:*", + "end-two-entity:*", + "end-one-entity-type:BusinessPolicy", + "end-two-entity-type:BusinessPolicy", + "relationship-type:*" + ], + "policyActions": + [ + "add-relationship", + "update-relationship", + "remove-relationship" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "LINK_BUSINESS_POLICY_BUSINESS_POLICY_EXCEPTION", + "qualifiedName": "LINK_BUSINESS_POLICY_BUSINESS_POLICY_EXCEPTION", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$api-token-default-access" + ], + "policyResourceCategory": "RELATIONSHIP", + "policyResources": + [ + "end-one-entity-classification:*", + "end-two-entity-classification:*", + "end-one-entity:*", + "end-two-entity:*", + "end-one-entity-type:BusinessPolicy", + "end-two-entity-type:BusinessPolicyException", + "relationship-type:*" + ], + "policyActions": + [ + "add-relationship", + "update-relationship", + "remove-relationship" + ] + } } ] } \ No newline at end of file From 850cf265e9dcfbb134d431c3569ff65530dcba5b Mon Sep 17 00:00:00 2001 From: arpit-at Date: Thu, 13 Jun 2024 17:53:32 +0530 Subject: [PATCH 316/316] DG-1432 Bootstrap relationship policies for MQ related Items --- addons/policies/bootstrap_entity_policies.json | 2 -- addons/policies/bootstrap_relationship_policies.json | 4 ++-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/addons/policies/bootstrap_entity_policies.json b/addons/policies/bootstrap_entity_policies.json index 23a77f5298e..38a6b86e80d 100644 --- a/addons/policies/bootstrap_entity_policies.json +++ b/addons/policies/bootstrap_entity_policies.json @@ -3190,7 +3190,6 @@ "policyResourceCategory": "ENTITY", "policyResources": [ "entity-type:Incident", - "entity-type:BusinessPolicyIncident", "entity-classification:*", "entity:*" ], @@ -3222,7 +3221,6 @@ "policyResourceCategory": "ENTITY", "policyResources": [ "entity-type:Incident", - "entity-type:BusinessPolicyIncident", "entity-classification:*", "entity:*" ], diff --git a/addons/policies/bootstrap_relationship_policies.json b/addons/policies/bootstrap_relationship_policies.json index 5ac53ccef3c..2c123bec6b5 100644 --- a/addons/policies/bootstrap_relationship_policies.json +++ b/addons/policies/bootstrap_relationship_policies.json @@ -789,7 +789,7 @@ "end-two-entity:*", "end-one-entity-type:BusinessPolicy", "end-two-entity-type:BusinessPolicy", - "relationship-type:*" + "relationship-type:RelatedBusinessPolicy" ], "policyActions": [ @@ -827,7 +827,7 @@ "end-two-entity:*", "end-one-entity-type:BusinessPolicy", "end-two-entity-type:BusinessPolicyException", - "relationship-type:*" + "relationship-type:BusinessPolicy_BusinessPolicyException" ], "policyActions": [