From 27285adcace0575077f373ed2161f0bd3d5e10f9 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Thu, 28 Mar 2024 10:02:29 +0530 Subject: [PATCH 01/25] Add DataContract preprocessor and versioning --- .../apache/atlas/repository/Constants.java | 5 + .../store/graph/v2/AtlasEntityStoreV2.java | 5 + .../AbstractContractPreProcessor.java | 129 +++++++++ .../contract/ContractPreProcessor.java | 245 ++++++++++++++++++ .../contract/ContractVersionUtils.java | 127 +++++++++ .../preprocessor/contract/DataContract.java | 129 +++++++++ 6 files changed, 640 insertions(+) create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 4a2cd5905c..9fdb249fcc 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -163,6 +163,11 @@ public final class Constants { public static final String ASSET_README_EDGE_LABEL = "__Asset.readme"; public static final String ASSET_LINK_EDGE_LABEL = "__Asset.links"; + /** + * Contract + */ + public static final String CONTRACT_ENTITY_TYPE = "DataContract"; + /** * Lineage relations. */ diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 6e3487f8d3..4daf9128da 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -58,6 +58,7 @@ import org.apache.atlas.repository.store.graph.v1.RestoreHandlerV1; import org.apache.atlas.repository.store.graph.v2.preprocessor.AuthPolicyPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.ConnectionPreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.contract.ContractPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.resource.LinkPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.accesscontrol.PersonaPreProcessor; @@ -1835,6 +1836,10 @@ public PreProcessor getPreProcessor(String typeName) { case README_ENTITY_TYPE: preProcessor = new ReadmePreProcessor(typeRegistry, entityRetriever); break; + + case CONTRACT_ENTITY_TYPE: + preProcessor = new ContractPreProcessor(graph, typeRegistry, entityRetriever, this); + break; } return preProcessor; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java new file mode 100644 index 0000000000..159dd94a4e --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java @@ -0,0 +1,129 @@ +package org.apache.atlas.repository.store.graph.v2.preprocessor.contract; + +import org.apache.atlas.RequestContext; +import org.apache.atlas.authorize.AtlasAuthorizationUtils; +import org.apache.atlas.authorize.AtlasEntityAccessRequest; +import org.apache.atlas.authorize.AtlasPrivilege; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.model.instance.AtlasObjectId; +import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.resource.AbstractResourcePreProcessor; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.Iterator; + +import static org.apache.atlas.repository.Constants.*; +import static org.apache.atlas.repository.Constants.ASSET_RELATION_ATTR; + +public abstract class AbstractContractPreProcessor implements PreProcessor { + private static final Logger LOG = LoggerFactory.getLogger(AbstractResourcePreProcessor.class); + + public final AtlasTypeRegistry typeRegistry; + public final EntityGraphRetriever entityRetriever; + public final AtlasGraph graph; + + + AbstractContractPreProcessor(AtlasGraph graph, AtlasTypeRegistry typeRegistry, + EntityGraphRetriever entityRetriever) { + this.graph = graph; + this.typeRegistry = typeRegistry; + this.entityRetriever = entityRetriever; + } + + void authorizeResourceUpdate(AtlasEntity resourceEntity, AtlasVertex ResourceVertex, String edgeLabel) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("authorizeResourceUpdate"); + + try { + AtlasEntityHeader assetEntity = null; + + AtlasObjectId asset = getAssetRelationAttr(resourceEntity); + if (asset != null) { + //Found linked asset in payload + AtlasVertex assetVertex = entityRetriever.getEntityVertex(asset); + assetEntity = entityRetriever.toAtlasEntityHeaderWithClassifications(assetVertex); + + } else { + //Check for linked asset in store + Iterator atlasVertexIterator = ResourceVertex.query() + .direction(AtlasEdgeDirection.IN) + .label(edgeLabel) + .has(STATE_PROPERTY_KEY, ACTIVE_STATE_VALUE) + .vertices() + .iterator(); + + if (atlasVertexIterator.hasNext()) { + //Found linked asset in store + AtlasVertex assetVertex = (AtlasVertex) atlasVertexIterator.next(); + assetEntity = entityRetriever.toAtlasEntityHeaderWithClassifications(assetVertex); + } + } + + if (assetEntity != null) { + //First authorize entity update access + verifyAssetAccess(assetEntity, AtlasPrivilege.ENTITY_UPDATE, resourceEntity, AtlasPrivilege.ENTITY_UPDATE); + } else { + //No linked asset to the Resource, check for resource update permission + verifyAccess(resourceEntity, AtlasPrivilege.ENTITY_UPDATE); + } + + } finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + } + + void authorizeResourceDelete(AtlasVertex resourceVertex) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("authorizeResourceDelete"); + + try { + AtlasEntity resourceEntity = entityRetriever.toAtlasEntity(resourceVertex); + + AtlasObjectId asset = getAssetRelationAttr(resourceEntity); + if (asset != null) { + AtlasEntityHeader assetEntity = entityRetriever.toAtlasEntityHeaderWithClassifications(asset.getGuid()); + verifyAssetAccess(assetEntity, AtlasPrivilege.ENTITY_UPDATE, resourceEntity, AtlasPrivilege.ENTITY_DELETE); + } else { + //No linked asset to the Resource, check for resource delete permission + verifyAccess(resourceEntity, AtlasPrivilege.ENTITY_DELETE); + } + } finally { + RequestContext.get().endMetricRecord(recorder); + } + } + + private AtlasObjectId getAssetRelationAttr(AtlasEntity entity) { + AtlasObjectId ret = null; + + if (entity.hasRelationshipAttribute(ASSET_RELATION_ATTR) && + entity.getRelationshipAttribute(ASSET_RELATION_ATTR) != null) { + ret = (AtlasObjectId) entity.getRelationshipAttribute(ASSET_RELATION_ATTR); + } + + return ret; + } + + private void verifyAssetAccess(AtlasEntityHeader asset, AtlasPrivilege assetPrivilege, + AtlasEntity resource, AtlasPrivilege resourcePrivilege) throws AtlasBaseException { + verifyAccess(asset, assetPrivilege); + verifyAccess(resource, resourcePrivilege); + } + + private void verifyAccess(AtlasEntity entity, AtlasPrivilege privilege) throws AtlasBaseException { + verifyAccess(new AtlasEntityHeader(entity), privilege); + } + + private void verifyAccess(AtlasEntityHeader entityHeader, AtlasPrivilege privilege) throws AtlasBaseException { + String errorMessage = privilege.name() + " entity: " + entityHeader.getTypeName(); + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, privilege, entityHeader), errorMessage); + } + + +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java new file mode 100644 index 0000000000..0b329ec47e --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -0,0 +1,245 @@ +package org.apache.atlas.repository.store.graph.v2.preprocessor.contract; + +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.exc.InvalidFormatException; +import com.sun.xml.bind.v2.TODO; +import joptsimple.internal.Strings; +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.RequestContext; +import org.apache.atlas.authorize.AtlasAuthorizationUtils; +import org.apache.atlas.authorize.AtlasEntityAccessRequest; +import org.apache.atlas.authorize.AtlasPrivilege; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.featureflag.FeatureFlagStore; +import org.apache.atlas.model.TypeCategory; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.model.instance.AtlasStruct; +import org.apache.atlas.model.instance.EntityMutations; +import org.apache.atlas.repository.Constants; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.AtlasEntityStore; +import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; +import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; +import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; +import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.resource.AbstractResourcePreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.resource.ReadmePreProcessor; +import org.apache.atlas.type.AtlasEntityType; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.commons.lang.StringUtils; +import org.jetbrains.annotations.NotNull; +import org.json.simple.JSONObject; +import org.json.simple.parser.JSONParser; +import org.json.simple.parser.ParseException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +//import org.pkl.config.java.Config; +//import org.pkl.config.java.ConfigEvaluator; +//import org.pkl.core.ModuleSource; +//import org.pkl.config.java.JavaType; + +//import org.pkl.codegen.java.JavaCodeGenerator; + +import com.fasterxml.jackson.databind.ObjectMapper; + + +import java.util.*; + +import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; +import static org.apache.atlas.repository.Constants.ASSET_README_EDGE_LABEL; +import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; +import static org.apache.atlas.repository.util.AccessControlUtils.*; +import static org.apache.atlas.repository.util.AccessControlUtils.getUUID; + +public class ContractPreProcessor extends AbstractContractPreProcessor { + private static final Logger LOG = LoggerFactory.getLogger(ReadmePreProcessor.class); + public static final String ATTR_CONTRACT = "contract"; + public static final String ATTR_CERTIFICATE_STATUS = "certificateStatus"; + private AtlasEntityStore entityStore; + + + public ContractPreProcessor(AtlasGraph graph, AtlasTypeRegistry typeRegistry, + EntityGraphRetriever entityRetriever, AtlasEntityStore entityStore) { + + super(graph, typeRegistry, entityRetriever); + this.entityStore = entityStore; + + } + + @Override + public void processAttributes(AtlasStruct entityStruct, EntityMutationContext context, EntityMutations.EntityOperation operation) throws AtlasBaseException { + AtlasEntity entity = (AtlasEntity) entityStruct; + switch (operation) { + case CREATE: + processCreateContract(entity, context); + break; + case UPDATE: + processUpdateContract(entity, context); + break; + } + + } + + private void processCreateContract(AtlasEntity entity, EntityMutationContext context) throws AtlasBaseException { + /* + Low-level Design + | Authorization + | Deserialization of the JSON + ---| Validation of spec + | Validation of contract + | Create Version + | Create Draft + ---| asset to contract sync + | Create Publish + ---| two-way sync of attribute + */ + + String contractString = (String) entity.getAttribute(ATTR_CONTRACT); + if (StringUtils.isEmpty(contractString)) { + throw new AtlasBaseException(BAD_REQUEST, "Please provide attribute " + ATTR_CONTRACT); + } + + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS); + DataContract contract; + try { + contract = objectMapper.readValue(contractString, DataContract.class); + } catch (Exception ex) { + ex.printStackTrace(); + throw new AtlasBaseException("Failed at this"); + } + + contractAttributeSync(entity, contract); + String contractQName = (String) entity.getAttribute(QUALIFIED_NAME); + validateAttribute(!contractQName.endsWith("/contract"), "Invalid qualifiedName for the contract."); + + String datasetQName = contractQName.substring(0, contractQName.lastIndexOf('/')); + ContractVersionUtils versionUtil = new ContractVersionUtils(entity, context, entityRetriever, typeRegistry, entityStore, graph); + AtlasEntity latestVersion = versionUtil.getLatestVersion(); + +// List vertexList = new ArrayList<>(); +// while (allContracts.hasNext()) { +// AtlasVertex v = allContracts.next(); +// vertexList.add(v); +// } +// RequestContext.get().endMetricRecord(metric); + + if (latestVersion != null) { + String qName = (String) latestVersion.getAttribute(QUALIFIED_NAME); + Integer latestVersionNumber = Integer.valueOf(qName.substring(qName.lastIndexOf("/V")+2)); + + entity.setAttribute(QUALIFIED_NAME, String.format("%s/version/V%s", contractQName, ++latestVersionNumber)); + + } else { + entity.setAttribute(QUALIFIED_NAME, String.format("%s/%s", contractQName, "version/V1")); + + } + + DataContract.Dataset dataset = contract.dataset; + AtlasEntityWithExtInfo associatedAsset = getAssociatedAsset(datasetQName, dataset.type.name()); + + if (contract.getStatus() == DataContract.STATUS.VERIFIED && + entity.getAttribute(ATTR_CERTIFICATE_STATUS).equals(DataContract.STATUS.VERIFIED.name())) { + datasetAttributeSync(associatedAsset.getEntity(), contract); + } + + +// versionUtil.createNewVersion(); + + } + + private void processUpdateContract(AtlasEntity entity, EntityMutationContext context) throws AtlasBaseException { + throw new AtlasBaseException("Can't update a specific version of contract"); + + + } + + private void contractAttributeSync(AtlasEntity entity, DataContract contract) { + // Sync certificateStatus + if (!Objects.equals(entity.getAttribute(ATTR_CERTIFICATE_STATUS), contract.getStatus().name())) { + /* + CertificateStatus | Status | Result + DRAFT VERIFIED cert -> VERIFIED > + VERIFIED DRAFT stat -> VERIFIED > + - DRAFT cert -> DRAFT + - VERIFIED cert -> VERIFIED > + DRAFT - stat -> DRAFT + VERIFIED - stat -> VERIFIED > + + */ + if (entity.getAttribute(ATTR_CERTIFICATE_STATUS) == DataContract.STATUS.VERIFIED.name()) { + contract.setStatus(DataContract.STATUS.VERIFIED); + } else if (contract.getStatus() == DataContract.STATUS.VERIFIED) { + entity.setAttribute(ATTR_CERTIFICATE_STATUS, DataContract.STATUS.VERIFIED.name()); + } else { + entity.setAttribute(ATTR_CERTIFICATE_STATUS, DataContract.STATUS.DRAFT); + contract.setStatus(DataContract.STATUS.DRAFT); + } + + } + + } + + private void datasetAttributeSync(AtlasEntity associatedAsset, DataContract contract) throws AtlasBaseException { + + DataContract.Dataset dataset = contract.dataset; + // Will implement dataset attribute sync from the contract attributes + + } + + private AtlasEntityWithExtInfo getLatestContract(String datasetQName) throws AtlasBaseException { + Map uniqAttributes = new HashMap<>(); + uniqAttributes.put(QUALIFIED_NAME, String.format("%s/%s", datasetQName, "contract/version/*")); + AtlasEntityType entityType = ensureEntityType(Constants.CONTRACT_ENTITY_TYPE); + + AtlasVertex entityVertex = AtlasGraphUtilsV2.getVertexByUniqueAttributes(graph, entityType, uniqAttributes); + + EntityGraphRetriever entityRetriever = new EntityGraphRetriever(graph, typeRegistry, true); + + AtlasEntityWithExtInfo ret = entityRetriever.toAtlasEntityWithExtInfo(entityVertex); + + if (ret == null) { + throw new AtlasBaseException(AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND, entityType.getTypeName(), + uniqAttributes.toString()); + } + return ret; + + } + private AtlasEntityWithExtInfo getAssociatedAsset(String qualifiedName, String typeName) throws AtlasBaseException { + Map uniqAttributes = new HashMap<>(); + uniqAttributes.put(QUALIFIED_NAME, qualifiedName); + + AtlasEntityType entityType = ensureEntityType(typeName); + + AtlasVertex entityVertex = AtlasGraphUtilsV2.getVertexByUniqueAttributes(graph, entityType, uniqAttributes); + + EntityGraphRetriever entityRetriever = new EntityGraphRetriever(graph, typeRegistry, true); + + AtlasEntityWithExtInfo ret = entityRetriever.toAtlasEntityWithExtInfo(entityVertex); + + if (ret == null) { + throw new AtlasBaseException(AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND, entityType.getTypeName(), + uniqAttributes.toString()); + } + return ret; + } + + private AtlasEntityType ensureEntityType(String typeName) throws AtlasBaseException { + AtlasEntityType ret = typeRegistry.getEntityTypeByName(typeName); + + if (ret == null) { + throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), typeName); + } + + return ret; + } + + private static void validateAttribute(boolean isInvalid, String errorMessage) throws AtlasBaseException { + if (isInvalid) + throw new AtlasBaseException(BAD_REQUEST, errorMessage); + } +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java new file mode 100644 index 0000000000..e476986ff3 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java @@ -0,0 +1,127 @@ +package org.apache.atlas.repository.store.graph.v2.preprocessor.contract; + +import org.apache.atlas.RequestContext; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.repository.graphdb.AtlasGraph; +import org.apache.atlas.repository.graphdb.AtlasGraphQuery; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.AtlasEntityStore; +import org.apache.atlas.repository.store.graph.v2.*; +import org.apache.atlas.repository.store.graph.v2.preprocessor.ConnectionPreProcessor; +import org.apache.atlas.type.AtlasEntityType; +import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; + +import static org.apache.atlas.repository.Constants.*; +import static org.apache.atlas.repository.graph.AtlasGraphProvider.getGraphInstance; + + +public class ContractVersionUtils { + private static final Logger LOG = LoggerFactory.getLogger(ConnectionPreProcessor.class); + + private final EntityMutationContext context; + public final EntityGraphRetriever entityRetriever; + private final AtlasTypeRegistry atlasTypeRegistry; + private AtlasEntityStore entityStore; + + private AtlasEntity entity; + private AtlasEntity existingEntity; + public final AtlasGraph graph; + private List versionList; + + + + public ContractVersionUtils(AtlasEntity entity, EntityMutationContext context, EntityGraphRetriever entityRetriever, + AtlasTypeRegistry atlasTypeRegistry, AtlasEntityStore entityStore, AtlasGraph graph) { + this.context = context; + this.entityRetriever = entityRetriever; + this.atlasTypeRegistry = atlasTypeRegistry; + this.graph = graph; + this.entityStore = entityStore; + this.entity = entity; + } + + private void extractAllVersions() { + String entityQNamePrefix = (String) entity.getAttribute(QUALIFIED_NAME); + + AtlasEntityType entityType = atlasTypeRegistry.getEntityTypeByName("DataContract"); + Integer versionCounter = 1; + boolean found = true; + + List versionList = new ArrayList<>(); + + while (found) { + Map uniqAttributes = new HashMap<>(); + uniqAttributes.put(QUALIFIED_NAME, String.format("%s/version/V%s", entityQNamePrefix, versionCounter++)); + try { + AtlasVertex entityVertex = AtlasGraphUtilsV2.getVertexByUniqueAttributes(graph, entityType, uniqAttributes); + AtlasEntity entity = entityRetriever.toAtlasEntity(entityVertex); + + versionList.add(entity); + } catch (AtlasBaseException ex) { + found = false; + } + + } + this.versionList = versionList; + + } + + public AtlasEntity getLatestVersion() throws AtlasBaseException { + if (this.versionList == null) { + extractAllVersions(); + } + Collections.sort(this.versionList, (e1, e2) -> { + String e1QName = (String) e1.getAttribute(QUALIFIED_NAME); + String e2QName = (String) e2.getAttribute(QUALIFIED_NAME); + + return e2QName.compareTo(e1QName); + }); + if (this.versionList.isEmpty()) { + return null; + } + return this.versionList.get(0); + } + + + public Iterator getAllEntityVersions() { + String entityQNamePrefix = (String) entity.getAttribute(QUALIFIED_NAME); + AtlasEntityType entityType = atlasTypeRegistry.getEntityTypeByName("DataContract"); + +// AtlasEntityType entityType, String name + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("getAllEntityVersions"); + AtlasGraph graph = getGraphInstance(); + AtlasGraphQuery query = graph.query() + .has(ENTITY_TYPE_PROPERTY_KEY, entityType.getTypeName()) + .has(STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name()) + .has(entityType.getAllAttributes().get(QUALIFIED_NAME).getQualifiedName(), String.format("%s/version/V1", entityQNamePrefix)); + + + Iterator result = query.vertices().iterator(); + + RequestContext.get().endMetricRecord(metric); + return result; + } + + public void createNewVersion() throws AtlasBaseException { + AtlasVertex vertex = context.getVertex(entity.getGuid()); + AtlasEntity existingContractEntity = entityRetriever.toAtlasEntity(vertex); +// this.newEntity = new AtlasEntity(existingEntity); + this.existingEntity.setAttribute(QUALIFIED_NAME, null); + + try { + RequestContext.get().setSkipAuthorizationCheck(true); + EntityStream entityStream = new AtlasEntityStream(existingEntity); + entityStore.createOrUpdate(entityStream, false); + LOG.info("Created bootstrap policies for connection {}", existingEntity.getAttribute(QUALIFIED_NAME)); + } finally { + RequestContext.get().setSkipAuthorizationCheck(false); + } + + } +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java new file mode 100644 index 0000000000..63da3e909f --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java @@ -0,0 +1,129 @@ +package org.apache.atlas.repository.store.graph.v2.preprocessor.contract; + +import java.lang.String; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonSetter; +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.commons.lang.StringUtils; + + +@JsonIgnoreProperties(ignoreUnknown = true) +public class DataContract { + @JsonProperty(required = true) + public String kind; + public STATUS status; + @JsonProperty(value = "template_version", defaultValue = "0.0.1") + public String templateVersion; + public Dataset dataset; + public List columns; + + public STATUS getStatus() { + return status; + } + + @JsonSetter("status") + public void setStatus(STATUS status) { + this.status = status; + } + + public enum STATUS { + @JsonProperty("DRAFT") DRAFT, + @JsonProperty("VERIFIED") VERIFIED; + + public static STATUS from(String s) { + if(StringUtils.isEmpty(s)) { + return DRAFT; + } + + switch (s.toLowerCase()) { + case "draft": + return DRAFT; + + case "verified": + return VERIFIED; + + default: + return DRAFT; + } + } + } + + @JsonSetter("kind") + public void setKind(String kind) throws AtlasBaseException { + if (!"DataContract".equals(kind)) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "kind " + kind + " is inappropriate."); + } + this.kind = kind; + } + + public void setTemplateVersion(String templateVersion) { + if (!isSemVer(templateVersion)) { + throw new IllegalArgumentException("Invalid version syntax"); + } + this.templateVersion = templateVersion; + } + + private boolean isSemVer(String version) { + Pattern versionPattern = Pattern.compile("^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$"); + Matcher matcher = versionPattern.matcher(version); + return matcher.matches(); + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static final class Dataset { + public String name; + @JsonProperty(required = true) + public DATASET_TYPE type; + public String description; + + + @JsonSetter("type") + public void setType(DATASET_TYPE type) { + this.type = type; + } + + public enum DATASET_TYPE { + @JsonProperty("Table") Table, + @JsonProperty("View") View, + @JsonProperty("MaterialisedView") MaterialisedView; + + public static DATASET_TYPE from(String s) throws AtlasBaseException { + + switch (s.toLowerCase()) { + case "table": + return Table; + case "view": + return View; + case "materialisedview": + return MaterialisedView; + default: + throw new AtlasBaseException("dataset.type value not supported yet."); + } + } + } + + + } + + @JsonIgnoreProperties(ignoreUnknown = true) + public static final class Column { + public String name; + + public String description; + + public boolean is_primary; + + public String data_type; + + + } + + +} + From 5c53ca11b3cbf279be2850b7e45bb80631826524 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Wed, 3 Apr 2024 10:15:52 +0530 Subject: [PATCH 02/25] Refactore contract code and release --- .github/workflows/maven.yml | 1 + .../apache/atlas/repository/Constants.java | 1 + .../store/graph/v2/AtlasEntityStoreV2.java | 2 +- .../AbstractContractPreProcessor.java | 120 ++++---- .../contract/ContractPreProcessor.java | 265 +++++++++--------- .../contract/ContractVersionUtils.java | 50 ++-- .../preprocessor/contract/DataContract.java | 30 ++ 7 files changed, 258 insertions(+), 211 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 0977cb36a2..39703359df 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,6 +26,7 @@ on: - development - master - lineageondemand + - data-contract jobs: build: diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 9fdb249fcc..925d0a6476 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -399,6 +399,7 @@ public enum SupportedFileExtensions { XLSX, XLS, CSV } public static final String ATTR_STARRED_DETAILS_LIST = "starredDetailsList"; public static final String ATTR_ASSET_STARRED_BY = "assetStarredBy"; public static final String ATTR_ASSET_STARRED_AT = "assetStarredAt"; + public static final String ATTR_CERTIFICATE_STATUS = "certificateStatus"; public static final String STRUCT_STARRED_DETAILS = "StarredDetails"; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 4daf9128da..02dd3965b8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -1838,7 +1838,7 @@ public PreProcessor getPreProcessor(String typeName) { break; case CONTRACT_ENTITY_TYPE: - preProcessor = new ContractPreProcessor(graph, typeRegistry, entityRetriever, this); + preProcessor = new ContractPreProcessor(graph, typeRegistry, entityRetriever, this, entityGraphMapper); break; } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java index 159dd94a4e..dd2e558156 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java @@ -5,27 +5,29 @@ import org.apache.atlas.authorize.AtlasEntityAccessRequest; import org.apache.atlas.authorize.AtlasPrivilege; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.TypeCategory; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntityHeader; -import org.apache.atlas.model.instance.AtlasObjectId; -import org.apache.atlas.repository.graphdb.AtlasEdgeDirection; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; -import org.apache.atlas.repository.store.graph.v2.preprocessor.resource.AbstractResourcePreProcessor; +import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.Iterator; +import java.util.HashMap; +import java.util.Map; +import static org.apache.atlas.AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND; +import static org.apache.atlas.AtlasErrorCode.TYPE_NAME_INVALID; import static org.apache.atlas.repository.Constants.*; -import static org.apache.atlas.repository.Constants.ASSET_RELATION_ATTR; public abstract class AbstractContractPreProcessor implements PreProcessor { - private static final Logger LOG = LoggerFactory.getLogger(AbstractResourcePreProcessor.class); + private static final Logger LOG = LoggerFactory.getLogger(AbstractContractPreProcessor.class); public final AtlasTypeRegistry typeRegistry; public final EntityGraphRetriever entityRetriever; @@ -39,81 +41,40 @@ public abstract class AbstractContractPreProcessor implements PreProcessor { this.entityRetriever = entityRetriever; } - void authorizeResourceUpdate(AtlasEntity resourceEntity, AtlasVertex ResourceVertex, String edgeLabel) throws AtlasBaseException { - AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("authorizeResourceUpdate"); - + void authorizeContractCreateOrUpdate(AtlasEntity contractEntity, AtlasEntity.AtlasEntityWithExtInfo associatedAsset) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("authorizeContractUpdate"); try { - AtlasEntityHeader assetEntity = null; - - AtlasObjectId asset = getAssetRelationAttr(resourceEntity); - if (asset != null) { - //Found linked asset in payload - AtlasVertex assetVertex = entityRetriever.getEntityVertex(asset); - assetEntity = entityRetriever.toAtlasEntityHeaderWithClassifications(assetVertex); - - } else { - //Check for linked asset in store - Iterator atlasVertexIterator = ResourceVertex.query() - .direction(AtlasEdgeDirection.IN) - .label(edgeLabel) - .has(STATE_PROPERTY_KEY, ACTIVE_STATE_VALUE) - .vertices() - .iterator(); - - if (atlasVertexIterator.hasNext()) { - //Found linked asset in store - AtlasVertex assetVertex = (AtlasVertex) atlasVertexIterator.next(); - assetEntity = entityRetriever.toAtlasEntityHeaderWithClassifications(assetVertex); - } - } - - if (assetEntity != null) { - //First authorize entity update access - verifyAssetAccess(assetEntity, AtlasPrivilege.ENTITY_UPDATE, resourceEntity, AtlasPrivilege.ENTITY_UPDATE); - } else { - //No linked asset to the Resource, check for resource update permission - verifyAccess(resourceEntity, AtlasPrivilege.ENTITY_UPDATE); - } + AtlasEntityHeader entityHeader = new AtlasEntityHeader(associatedAsset.getEntity()); + + //First authorize entity update access + verifyAssetAccess(entityHeader, AtlasPrivilege.ENTITY_UPDATE, contractEntity, AtlasPrivilege.ENTITY_UPDATE); } finally { RequestContext.get().endMetricRecord(metricRecorder); } } - void authorizeResourceDelete(AtlasVertex resourceVertex) throws AtlasBaseException { - AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("authorizeResourceDelete"); + + void authorizeContractDelete(AtlasVertex contractVertex, String typeName) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("authorizeContractDelete"); try { - AtlasEntity resourceEntity = entityRetriever.toAtlasEntity(resourceVertex); - - AtlasObjectId asset = getAssetRelationAttr(resourceEntity); - if (asset != null) { - AtlasEntityHeader assetEntity = entityRetriever.toAtlasEntityHeaderWithClassifications(asset.getGuid()); - verifyAssetAccess(assetEntity, AtlasPrivilege.ENTITY_UPDATE, resourceEntity, AtlasPrivilege.ENTITY_DELETE); - } else { - //No linked asset to the Resource, check for resource delete permission - verifyAccess(resourceEntity, AtlasPrivilege.ENTITY_DELETE); - } + AtlasEntity contractEntity = entityRetriever.toAtlasEntity(contractVertex); + String contractQName = contractEntity.getAttribute(QUALIFIED_NAME).toString(); + AtlasEntity.AtlasEntityWithExtInfo assetEntity = getAssociatedAsset(contractQName, typeName); + AtlasEntityHeader entityHeader = new AtlasEntityHeader(assetEntity.getEntity()); + + verifyAssetAccess(entityHeader, AtlasPrivilege.ENTITY_UPDATE, contractEntity, AtlasPrivilege.ENTITY_DELETE); } finally { RequestContext.get().endMetricRecord(recorder); } } - private AtlasObjectId getAssetRelationAttr(AtlasEntity entity) { - AtlasObjectId ret = null; - - if (entity.hasRelationshipAttribute(ASSET_RELATION_ATTR) && - entity.getRelationshipAttribute(ASSET_RELATION_ATTR) != null) { - ret = (AtlasObjectId) entity.getRelationshipAttribute(ASSET_RELATION_ATTR); - } - - return ret; - } private void verifyAssetAccess(AtlasEntityHeader asset, AtlasPrivilege assetPrivilege, - AtlasEntity resource, AtlasPrivilege resourcePrivilege) throws AtlasBaseException { + AtlasEntity contract, AtlasPrivilege contractPrivilege) throws AtlasBaseException { verifyAccess(asset, assetPrivilege); - verifyAccess(resource, resourcePrivilege); + verifyAccess(contract, contractPrivilege); } private void verifyAccess(AtlasEntity entity, AtlasPrivilege privilege) throws AtlasBaseException { @@ -125,5 +86,36 @@ private void verifyAccess(AtlasEntityHeader entityHeader, AtlasPrivilege privile AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, privilege, entityHeader), errorMessage); } + AtlasEntity.AtlasEntityWithExtInfo getAssociatedAsset(String contractQName, String typeName) throws AtlasBaseException { + String datasetQName = contractQName.substring(0, contractQName.lastIndexOf('/')); + + Map uniqAttributes = new HashMap<>(); + uniqAttributes.put(QUALIFIED_NAME, datasetQName); + + AtlasEntityType entityType = ensureEntityType(typeName); + + AtlasVertex entityVertex = AtlasGraphUtilsV2.getVertexByUniqueAttributes(graph, entityType, uniqAttributes); + + EntityGraphRetriever entityRetriever = new EntityGraphRetriever(graph, typeRegistry, true); + + AtlasEntity.AtlasEntityWithExtInfo ret = entityRetriever.toAtlasEntityWithExtInfo(entityVertex); + + if (ret == null) { + throw new AtlasBaseException(INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND, entityType.getTypeName(), + uniqAttributes.toString()); + } + return ret; + } + + AtlasEntityType ensureEntityType(String typeName) throws AtlasBaseException { + AtlasEntityType ret = typeRegistry.getEntityTypeByName(typeName); + + if (ret == null) { + throw new AtlasBaseException(TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), typeName); + } + + return ret; + } + } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index 0b329ec47e..aecca4ea9d 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -1,73 +1,48 @@ package org.apache.atlas.repository.store.graph.v2.preprocessor.contract; -import com.fasterxml.jackson.databind.MapperFeature; -import com.fasterxml.jackson.databind.exc.InvalidFormatException; -import com.sun.xml.bind.v2.TODO; -import joptsimple.internal.Strings; -import org.apache.atlas.AtlasErrorCode; -import org.apache.atlas.RequestContext; -import org.apache.atlas.authorize.AtlasAuthorizationUtils; -import org.apache.atlas.authorize.AtlasEntityAccessRequest; -import org.apache.atlas.authorize.AtlasPrivilege; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.node.ObjectNode; + import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.featureflag.FeatureFlagStore; -import org.apache.atlas.model.TypeCategory; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; -import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasStruct; import org.apache.atlas.model.instance.EntityMutations; -import org.apache.atlas.repository.Constants; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.AtlasEntityStore; -import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; -import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; -import org.apache.atlas.repository.store.graph.v2.EntityMutationContext; -import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; -import org.apache.atlas.repository.store.graph.v2.preprocessor.resource.AbstractResourcePreProcessor; -import org.apache.atlas.repository.store.graph.v2.preprocessor.resource.ReadmePreProcessor; +import org.apache.atlas.repository.store.graph.v2.*; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; -import org.apache.commons.lang.StringUtils; -import org.jetbrains.annotations.NotNull; -import org.json.simple.JSONObject; -import org.json.simple.parser.JSONParser; -import org.json.simple.parser.ParseException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -//import org.pkl.config.java.Config; -//import org.pkl.config.java.ConfigEvaluator; -//import org.pkl.core.ModuleSource; -//import org.pkl.config.java.JavaType; - -//import org.pkl.codegen.java.JavaCodeGenerator; - import com.fasterxml.jackson.databind.ObjectMapper; - import java.util.*; -import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; -import static org.apache.atlas.repository.Constants.ASSET_README_EDGE_LABEL; +import static org.apache.atlas.AtlasErrorCode.*; +import static org.apache.atlas.repository.Constants.ATTR_CERTIFICATE_STATUS; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; -import static org.apache.atlas.repository.util.AccessControlUtils.*; -import static org.apache.atlas.repository.util.AccessControlUtils.getUUID; public class ContractPreProcessor extends AbstractContractPreProcessor { - private static final Logger LOG = LoggerFactory.getLogger(ReadmePreProcessor.class); - public static final String ATTR_CONTRACT = "contract"; - public static final String ATTR_CERTIFICATE_STATUS = "certificateStatus"; - private AtlasEntityStore entityStore; + private static final Logger LOG = LoggerFactory.getLogger(ContractPreProcessor.class); + public static final String ATTR_CONTRACT = "contract"; + public static final String CONTRACT_QUALIFIED_NAME_SUFFIX = "contract"; + public static final String VERSION_PREFIX = "version"; + public static final String CONTRACT_ATTR_STATUS = "status"; + private final AtlasEntityStore entityStore; + private final EntityGraphMapper entityGraphMapper; public ContractPreProcessor(AtlasGraph graph, AtlasTypeRegistry typeRegistry, - EntityGraphRetriever entityRetriever, AtlasEntityStore entityStore) { + EntityGraphRetriever entityRetriever, AtlasEntityStore entityStore, EntityGraphMapper entityGraphMapper) { super(graph, typeRegistry, entityRetriever); this.entityStore = entityStore; - + this.entityGraphMapper = entityGraphMapper; } @Override @@ -79,7 +54,49 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co break; case UPDATE: processUpdateContract(entity, context); - break; + } + + } + + private void processUpdateContract(AtlasEntity entity, EntityMutationContext context) throws AtlasBaseException { + String contractString = (String) entity.getAttribute(ATTR_CONTRACT); + AtlasVertex vertex = context.getVertex(entity.getGuid()); + AtlasEntity existingContractEntity = entityRetriever.toAtlasEntity(vertex); + + if (!isEqualContract(contractString, (String) existingContractEntity.getAttribute(ATTR_CONTRACT))) { + // Update the same asset(entity) + throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Can't update a specific version of contract"); + } + // Add cases for update in status field and certificateStatus + } + + private List getDiffAttributes(EntityMutationContext context, AtlasEntity entity, AtlasEntity latestExistingVersion) throws AtlasBaseException { + AtlasEntityComparator entityComparator = new AtlasEntityComparator(typeRegistry, entityRetriever, context.getGuidAssignments(), true, true); + AtlasEntityComparator.AtlasEntityDiffResult diffResult = entityComparator.getDiffResult(entity, latestExistingVersion, false); + List attributesSet = new ArrayList<>(); + + if (diffResult.hasDifference()) { + for (Map.Entry entry : diffResult.getDiffEntity().getAttributes().entrySet()) { + if (!entry.getKey().equals(QUALIFIED_NAME)) { + attributesSet.add(entry.getKey()); + } + } + } + return attributesSet; + } + + private boolean isEqualContract(String firstNode, String secondNode) throws AtlasBaseException { + ObjectMapper mapper = new ObjectMapper(); + try { + JsonNode actualObj1 = mapper.readTree(firstNode); + JsonNode actualObj2 = mapper.readTree(secondNode); + //Ignore status field change + ((ObjectNode) actualObj1).remove(CONTRACT_ATTR_STATUS); + ((ObjectNode) actualObj2).remove(CONTRACT_ATTR_STATUS); + + return actualObj1.equals(actualObj2); + } catch (JsonProcessingException e) { + throw new AtlasBaseException(JSON_ERROR, e.getMessage()); } } @@ -98,63 +115,104 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con ---| two-way sync of attribute */ + String contractQName = (String) entity.getAttribute(QUALIFIED_NAME); String contractString = (String) entity.getAttribute(ATTR_CONTRACT); - if (StringUtils.isEmpty(contractString)) { - throw new AtlasBaseException(BAD_REQUEST, "Please provide attribute " + ATTR_CONTRACT); - } + DataContract contract = DataContract.deserialize(contractString); + DataContract.Dataset dataset = contract.dataset; + AtlasEntityWithExtInfo associatedAsset = getAssociatedAsset(contractQName, dataset.type.name()); - ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS); - DataContract contract; - try { - contract = objectMapper.readValue(contractString, DataContract.class); - } catch (Exception ex) { - ex.printStackTrace(); - throw new AtlasBaseException("Failed at this"); - } + authorizeContractCreateOrUpdate(entity, associatedAsset); contractAttributeSync(entity, contract); - String contractQName = (String) entity.getAttribute(QUALIFIED_NAME); - validateAttribute(!contractQName.endsWith("/contract"), "Invalid qualifiedName for the contract."); - - String datasetQName = contractQName.substring(0, contractQName.lastIndexOf('/')); - ContractVersionUtils versionUtil = new ContractVersionUtils(entity, context, entityRetriever, typeRegistry, entityStore, graph); - AtlasEntity latestVersion = versionUtil.getLatestVersion(); + contractString = updateContract(contract, contractString); + entity.setAttribute(ATTR_CONTRACT, contractString); -// List vertexList = new ArrayList<>(); -// while (allContracts.hasNext()) { -// AtlasVertex v = allContracts.next(); -// vertexList.add(v); -// } -// RequestContext.get().endMetricRecord(metric); + validateAttribute(!contractQName.endsWith(String.format("/%s", CONTRACT_QUALIFIED_NAME_SUFFIX)), "Invalid qualifiedName for the contract."); - if (latestVersion != null) { - String qName = (String) latestVersion.getAttribute(QUALIFIED_NAME); - Integer latestVersionNumber = Integer.valueOf(qName.substring(qName.lastIndexOf("/V")+2)); + ContractVersionUtils versionUtil = new ContractVersionUtils(entity, context, entityRetriever, typeRegistry, entityStore, graph); + AtlasEntity latestExistingVersion = versionUtil.getLatestVersion(); + + if (latestExistingVersion != null) { + // Contract already exist + String qName = (String) latestExistingVersion.getAttribute(QUALIFIED_NAME); + Integer latestVersionNumber = Integer.valueOf(qName.substring(qName.lastIndexOf("/V") + 2)); + List attributes = getDiffAttributes(context, entity, latestExistingVersion); + if (attributes.isEmpty()) { + context.getCreatedEntities().remove(entity); + Set guids = new HashSet<>(); + guids.add(entity.getGuid()); + entityStore.purgeByIds(guids); + return; + } - entity.setAttribute(QUALIFIED_NAME, String.format("%s/version/V%s", contractQName, ++latestVersionNumber)); + if (attributes.size() == 1 && attributes.contains(ATTR_CERTIFICATE_STATUS)) { + if (Objects.equals(entity.getAttribute(ATTR_CERTIFICATE_STATUS).toString(), DataContract.STATUS.VERIFIED.name())) { + //update existing entity + updateExistingVersion(context, entity, latestExistingVersion); + } + // Contract status changed, either to draft or verified + return; + } else if (attributes.contains(ATTR_CONTRACT)) { + //Contract is changed + if (isEqualContract(contractString, (String) latestExistingVersion.getAttribute(ATTR_CONTRACT))) { + // Update the same asset(entity) + updateExistingVersion(context, entity, latestExistingVersion); + + } else { + // Create New version of entity + entity.setAttribute(QUALIFIED_NAME, String.format("%s/%s/V%s", contractQName, VERSION_PREFIX, ++latestVersionNumber)); + + } + + return; + } } else { - entity.setAttribute(QUALIFIED_NAME, String.format("%s/%s", contractQName, "version/V1")); + // Create new contract + entity.setAttribute(QUALIFIED_NAME, String.format("%s/%s/%s", contractQName, VERSION_PREFIX, "V1")); } - DataContract.Dataset dataset = contract.dataset; - AtlasEntityWithExtInfo associatedAsset = getAssociatedAsset(datasetQName, dataset.type.name()); - + entity.setAttribute("assetQualifiedName", associatedAsset.getEntity().getAttribute(QUALIFIED_NAME)); if (contract.getStatus() == DataContract.STATUS.VERIFIED && entity.getAttribute(ATTR_CERTIFICATE_STATUS).equals(DataContract.STATUS.VERIFIED.name())) { datasetAttributeSync(associatedAsset.getEntity(), contract); } + } -// versionUtil.createNewVersion(); + private String updateContract(DataContract contract, String contractString) throws AtlasBaseException { + ObjectMapper objectMapper = new ObjectMapper(); + try { + String contractObjString = DataContract.serialize(contract); + Map mapJson = objectMapper.readValue(contractString, new TypeReference>() { + }); + Map mapTargetJson = objectMapper.readValue(contractObjString, new TypeReference>() { + }); + + for (Map.Entry keyVal : mapTargetJson.entrySet()) { + mapJson.replace(keyVal.getKey(), keyVal.getValue()); + } + return objectMapper.writeValueAsString(mapJson); + } catch (JsonProcessingException ex) { + throw new AtlasBaseException(JSON_ERROR, ex.getMessage()); + } } - private void processUpdateContract(AtlasEntity entity, EntityMutationContext context) throws AtlasBaseException { - throw new AtlasBaseException("Can't update a specific version of contract"); + private void updateExistingVersion(EntityMutationContext context, AtlasEntity entity, AtlasEntity latestExistingVersion) throws AtlasBaseException { + context.getCreatedEntities().remove(entity); + Set guids = new HashSet<>(); + guids.add(entity.getGuid()); + entityStore.purgeByIds(guids); + entity.setAttribute(QUALIFIED_NAME, latestExistingVersion.getAttribute(QUALIFIED_NAME)); + entity.setGuid(latestExistingVersion.getGuid()); + + AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(entity.getGuid()); + + AtlasEntityType entityType = ensureEntityType(entity.getTypeName()); + context.addUpdated(entity.getGuid(), entity, entityType, vertex); } @@ -171,9 +229,9 @@ private void contractAttributeSync(AtlasEntity entity, DataContract contract) { VERIFIED - stat -> VERIFIED > */ - if (entity.getAttribute(ATTR_CERTIFICATE_STATUS) == DataContract.STATUS.VERIFIED.name()) { + if (Objects.equals(entity.getAttribute(ATTR_CERTIFICATE_STATUS), DataContract.STATUS.VERIFIED.name())) { contract.setStatus(DataContract.STATUS.VERIFIED); - } else if (contract.getStatus() == DataContract.STATUS.VERIFIED) { + } else if (Objects.equals(contract.getStatus(), DataContract.STATUS.VERIFIED)) { entity.setAttribute(ATTR_CERTIFICATE_STATUS, DataContract.STATUS.VERIFIED.name()); } else { entity.setAttribute(ATTR_CERTIFICATE_STATUS, DataContract.STATUS.DRAFT); @@ -191,53 +249,6 @@ private void datasetAttributeSync(AtlasEntity associatedAsset, DataContract cont } - private AtlasEntityWithExtInfo getLatestContract(String datasetQName) throws AtlasBaseException { - Map uniqAttributes = new HashMap<>(); - uniqAttributes.put(QUALIFIED_NAME, String.format("%s/%s", datasetQName, "contract/version/*")); - AtlasEntityType entityType = ensureEntityType(Constants.CONTRACT_ENTITY_TYPE); - - AtlasVertex entityVertex = AtlasGraphUtilsV2.getVertexByUniqueAttributes(graph, entityType, uniqAttributes); - - EntityGraphRetriever entityRetriever = new EntityGraphRetriever(graph, typeRegistry, true); - - AtlasEntityWithExtInfo ret = entityRetriever.toAtlasEntityWithExtInfo(entityVertex); - - if (ret == null) { - throw new AtlasBaseException(AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND, entityType.getTypeName(), - uniqAttributes.toString()); - } - return ret; - - } - private AtlasEntityWithExtInfo getAssociatedAsset(String qualifiedName, String typeName) throws AtlasBaseException { - Map uniqAttributes = new HashMap<>(); - uniqAttributes.put(QUALIFIED_NAME, qualifiedName); - - AtlasEntityType entityType = ensureEntityType(typeName); - - AtlasVertex entityVertex = AtlasGraphUtilsV2.getVertexByUniqueAttributes(graph, entityType, uniqAttributes); - - EntityGraphRetriever entityRetriever = new EntityGraphRetriever(graph, typeRegistry, true); - - AtlasEntityWithExtInfo ret = entityRetriever.toAtlasEntityWithExtInfo(entityVertex); - - if (ret == null) { - throw new AtlasBaseException(AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND, entityType.getTypeName(), - uniqAttributes.toString()); - } - return ret; - } - - private AtlasEntityType ensureEntityType(String typeName) throws AtlasBaseException { - AtlasEntityType ret = typeRegistry.getEntityTypeByName(typeName); - - if (ret == null) { - throw new AtlasBaseException(AtlasErrorCode.TYPE_NAME_INVALID, TypeCategory.ENTITY.name(), typeName); - } - - return ret; - } - private static void validateAttribute(boolean isInvalid, String errorMessage) throws AtlasBaseException { if (isInvalid) throw new AtlasBaseException(BAD_REQUEST, errorMessage); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java index e476986ff3..8d27a10bc1 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java @@ -1,5 +1,6 @@ package org.apache.atlas.repository.store.graph.v2.preprocessor.contract; +import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.RequestContext; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.AtlasEntity; @@ -30,7 +31,7 @@ public class ContractVersionUtils { private AtlasEntityStore entityStore; private AtlasEntity entity; - private AtlasEntity existingEntity; + public final AtlasGraph graph; private List versionList; @@ -47,7 +48,8 @@ public ContractVersionUtils(AtlasEntity entity, EntityMutationContext context, E } private void extractAllVersions() { - String entityQNamePrefix = (String) entity.getAttribute(QUALIFIED_NAME); + String contractQName = (String) entity.getAttribute(QUALIFIED_NAME); + String datasetQName = contractQName.substring(0, contractQName.lastIndexOf("/contract")); AtlasEntityType entityType = atlasTypeRegistry.getEntityTypeByName("DataContract"); Integer versionCounter = 1; @@ -57,10 +59,20 @@ private void extractAllVersions() { while (found) { Map uniqAttributes = new HashMap<>(); - uniqAttributes.put(QUALIFIED_NAME, String.format("%s/version/V%s", entityQNamePrefix, versionCounter++)); + uniqAttributes.put(QUALIFIED_NAME, String.format("%s/contract/version/V%s", datasetQName, versionCounter++)); try { AtlasVertex entityVertex = AtlasGraphUtilsV2.getVertexByUniqueAttributes(graph, entityType, uniqAttributes); AtlasEntity entity = entityRetriever.toAtlasEntity(entityVertex); +// +// EntityGraphRetriever entityRetriever = new EntityGraphRetriever(graph, typeRegistry, true); +// +// AtlasEntity.AtlasEntityWithExtInfo ret = entityRetriever.toAtlasEntityWithExtInfo(entityVertex); +// +// if (ret == null) { +// throw new AtlasBaseException(AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND, entityType.getTypeName(), +// uniqAttributes.toString()); +// } +// return ret; versionList.add(entity); } catch (AtlasBaseException ex) { @@ -108,20 +120,20 @@ public Iterator getAllEntityVersions() { return result; } - public void createNewVersion() throws AtlasBaseException { - AtlasVertex vertex = context.getVertex(entity.getGuid()); - AtlasEntity existingContractEntity = entityRetriever.toAtlasEntity(vertex); -// this.newEntity = new AtlasEntity(existingEntity); - this.existingEntity.setAttribute(QUALIFIED_NAME, null); - - try { - RequestContext.get().setSkipAuthorizationCheck(true); - EntityStream entityStream = new AtlasEntityStream(existingEntity); - entityStore.createOrUpdate(entityStream, false); - LOG.info("Created bootstrap policies for connection {}", existingEntity.getAttribute(QUALIFIED_NAME)); - } finally { - RequestContext.get().setSkipAuthorizationCheck(false); - } - - } +// public void createNewVersion() throws AtlasBaseException { +// AtlasVertex vertex = context.getVertex(entity.getGuid()); +// AtlasEntity existingContractEntity = entityRetriever.toAtlasEntity(vertex); +//// this.newEntity = new AtlasEntity(existingEntity); +// this.existingEntity.setAttribute(QUALIFIED_NAME, null); +// +// try { +// RequestContext.get().setSkipAuthorizationCheck(true); +// EntityStream entityStream = new AtlasEntityStream(existingEntity); +// entityStore.createOrUpdate(entityStream, false); +// LOG.info("Created bootstrap policies for connection {}", existingEntity.getAttribute(QUALIFIED_NAME)); +// } finally { +// RequestContext.get().setSkipAuthorizationCheck(false); +// } +// +// } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java index 63da3e909f..e6d68f3895 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java @@ -8,10 +8,15 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties; import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonSetter; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.MapperFeature; +import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.exception.AtlasBaseException; import org.apache.commons.lang.StringUtils; +import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; + @JsonIgnoreProperties(ignoreUnknown = true) public class DataContract { @@ -124,6 +129,31 @@ public static final class Column { } + public static DataContract deserialize(String contractString) throws AtlasBaseException { + + if (StringUtils.isEmpty(contractString)) { + throw new AtlasBaseException(BAD_REQUEST, "Missing attribute: contract."); + } + + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS); + DataContract contract; + try { + contract = objectMapper.readValue(contractString, DataContract.class); + } catch (Exception ex) { + ex.printStackTrace(); + throw new AtlasBaseException("Failed at this"); + } + return contract; + } + + public static String serialize(DataContract contract) throws JsonProcessingException { + + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS); + return objectMapper.writeValueAsString(contract); + } + } From 94d98db894fd146166eddbcbfc529bfb1dca3e13 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Thu, 4 Apr 2024 11:04:03 +0530 Subject: [PATCH 03/25] Add unknown attributes to DataContract and fix minor bugs --- .../store/graph/v2/AtlasEntityStoreV2.java | 5 +- .../contract/ContractPreProcessor.java | 112 +++++++++--------- .../preprocessor/contract/DataContract.java | 55 +++++++-- 3 files changed, 105 insertions(+), 67 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 02dd3965b8..2b2c6a7a3c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -695,8 +695,9 @@ public EntityMutationResponse purgeByIds(Set guids) throws AtlasBaseExce if (CollectionUtils.isEmpty(guids)) { throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "Guid(s) not specified"); } - - AtlasAuthorizationUtils.verifyAccess(new AtlasAdminAccessRequest(AtlasPrivilege.ADMIN_PURGE), "purge entity: guids=", guids); + if (!RequestContext.get().isSkipAuthorizationCheck()) { + AtlasAuthorizationUtils.verifyAccess(new AtlasAdminAccessRequest(AtlasPrivilege.ADMIN_PURGE), "purge entity: guids=", guids); + } Collection purgeCandidates = new ArrayList<>(); for (String guid : guids) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index aecca4ea9d..adfe3e4e2a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -5,10 +5,12 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; +import org.apache.atlas.RequestContext; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; import org.apache.atlas.model.instance.AtlasStruct; +import org.apache.atlas.model.instance.EntityMutationResponse; import org.apache.atlas.model.instance.EntityMutations; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; @@ -30,6 +32,9 @@ public class ContractPreProcessor extends AbstractContractPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(ContractPreProcessor.class); public static final String ATTR_CONTRACT = "contract"; + public static final String ATTR_VERSION = "contractVersion"; + public static final String ATTR_ASSET_QUALIFIED_NAME = "contractAssetQualifiedName"; + public static final String ATTR_PARENT_GUID = "parentGuid"; public static final String CONTRACT_QUALIFIED_NAME_SUFFIX = "contract"; public static final String VERSION_PREFIX = "version"; public static final String CONTRACT_ATTR_STATUS = "status"; @@ -53,6 +58,7 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co processCreateContract(entity, context); break; case UPDATE: + // Updating an existing version of the contract processUpdateContract(entity, context); } @@ -69,38 +75,6 @@ private void processUpdateContract(AtlasEntity entity, EntityMutationContext con } // Add cases for update in status field and certificateStatus } - - private List getDiffAttributes(EntityMutationContext context, AtlasEntity entity, AtlasEntity latestExistingVersion) throws AtlasBaseException { - AtlasEntityComparator entityComparator = new AtlasEntityComparator(typeRegistry, entityRetriever, context.getGuidAssignments(), true, true); - AtlasEntityComparator.AtlasEntityDiffResult diffResult = entityComparator.getDiffResult(entity, latestExistingVersion, false); - List attributesSet = new ArrayList<>(); - - if (diffResult.hasDifference()) { - for (Map.Entry entry : diffResult.getDiffEntity().getAttributes().entrySet()) { - if (!entry.getKey().equals(QUALIFIED_NAME)) { - attributesSet.add(entry.getKey()); - } - } - } - return attributesSet; - } - - private boolean isEqualContract(String firstNode, String secondNode) throws AtlasBaseException { - ObjectMapper mapper = new ObjectMapper(); - try { - JsonNode actualObj1 = mapper.readTree(firstNode); - JsonNode actualObj2 = mapper.readTree(secondNode); - //Ignore status field change - ((ObjectNode) actualObj1).remove(CONTRACT_ATTR_STATUS); - ((ObjectNode) actualObj2).remove(CONTRACT_ATTR_STATUS); - - return actualObj1.equals(actualObj2); - } catch (JsonProcessingException e) { - throw new AtlasBaseException(JSON_ERROR, e.getMessage()); - } - - } - private void processCreateContract(AtlasEntity entity, EntityMutationContext context) throws AtlasBaseException { /* Low-level Design @@ -116,6 +90,8 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con */ String contractQName = (String) entity.getAttribute(QUALIFIED_NAME); + validateAttribute(!contractQName.endsWith(String.format("/%s", CONTRACT_QUALIFIED_NAME_SUFFIX)), "Invalid qualifiedName for the contract."); + String contractString = (String) entity.getAttribute(ATTR_CONTRACT); DataContract contract = DataContract.deserialize(contractString); DataContract.Dataset dataset = contract.dataset; @@ -124,10 +100,9 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con authorizeContractCreateOrUpdate(entity, associatedAsset); contractAttributeSync(entity, contract); - contractString = updateContract(contract, contractString); + contractString = DataContract.serialize(contract); entity.setAttribute(ATTR_CONTRACT, contractString); - validateAttribute(!contractQName.endsWith(String.format("/%s", CONTRACT_QUALIFIED_NAME_SUFFIX)), "Invalid qualifiedName for the contract."); ContractVersionUtils versionUtil = new ContractVersionUtils(entity, context, entityRetriever, typeRegistry, entityStore, graph); AtlasEntity latestExistingVersion = versionUtil.getLatestVersion(); @@ -138,10 +113,7 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con Integer latestVersionNumber = Integer.valueOf(qName.substring(qName.lastIndexOf("/V") + 2)); List attributes = getDiffAttributes(context, entity, latestExistingVersion); if (attributes.isEmpty()) { - context.getCreatedEntities().remove(entity); - Set guids = new HashSet<>(); - guids.add(entity.getGuid()); - entityStore.purgeByIds(guids); + removeCreatingVertex(context, entity); return; } @@ -161,6 +133,9 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con } else { // Create New version of entity entity.setAttribute(QUALIFIED_NAME, String.format("%s/%s/V%s", contractQName, VERSION_PREFIX, ++latestVersionNumber)); + entity.setAttribute(ATTR_VERSION, String.format("V%s", latestVersionNumber)); + entity.setAttribute(ATTR_ASSET_QUALIFIED_NAME, associatedAsset.getEntity().getAttribute(QUALIFIED_NAME)); + entity.setAttribute(ATTR_PARENT_GUID, latestExistingVersion.getGuid()); } @@ -170,10 +145,11 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con } else { // Create new contract entity.setAttribute(QUALIFIED_NAME, String.format("%s/%s/%s", contractQName, VERSION_PREFIX, "V1")); + entity.setAttribute(ATTR_VERSION, "V1"); + entity.setAttribute(ATTR_ASSET_QUALIFIED_NAME, associatedAsset.getEntity().getAttribute(QUALIFIED_NAME)); } - entity.setAttribute("assetQualifiedName", associatedAsset.getEntity().getAttribute(QUALIFIED_NAME)); if (contract.getStatus() == DataContract.STATUS.VERIFIED && entity.getAttribute(ATTR_CERTIFICATE_STATUS).equals(DataContract.STATUS.VERIFIED.name())) { datasetAttributeSync(associatedAsset.getEntity(), contract); @@ -181,30 +157,39 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con } - private String updateContract(DataContract contract, String contractString) throws AtlasBaseException { - ObjectMapper objectMapper = new ObjectMapper(); - try { - String contractObjString = DataContract.serialize(contract); - Map mapJson = objectMapper.readValue(contractString, new TypeReference>() { - }); - Map mapTargetJson = objectMapper.readValue(contractObjString, new TypeReference>() { - }); - - for (Map.Entry keyVal : mapTargetJson.entrySet()) { - mapJson.replace(keyVal.getKey(), keyVal.getValue()); + private List getDiffAttributes(EntityMutationContext context, AtlasEntity entity, AtlasEntity latestExistingVersion) throws AtlasBaseException { + AtlasEntityComparator entityComparator = new AtlasEntityComparator(typeRegistry, entityRetriever, context.getGuidAssignments(), true, true); + AtlasEntityComparator.AtlasEntityDiffResult diffResult = entityComparator.getDiffResult(entity, latestExistingVersion, false); + List attributesSet = new ArrayList<>(); + + if (diffResult.hasDifference()) { + for (Map.Entry entry : diffResult.getDiffEntity().getAttributes().entrySet()) { + if (!entry.getKey().equals(QUALIFIED_NAME)) { + attributesSet.add(entry.getKey()); + } } - return objectMapper.writeValueAsString(mapJson); - } catch (JsonProcessingException ex) { - throw new AtlasBaseException(JSON_ERROR, ex.getMessage()); + } + return attributesSet; + } + + private boolean isEqualContract(String firstNode, String secondNode) throws AtlasBaseException { + ObjectMapper mapper = new ObjectMapper(); + try { + JsonNode actualObj1 = mapper.readTree(firstNode); + JsonNode actualObj2 = mapper.readTree(secondNode); + //Ignore status field change + ((ObjectNode) actualObj1).remove(CONTRACT_ATTR_STATUS); + ((ObjectNode) actualObj2).remove(CONTRACT_ATTR_STATUS); + + return actualObj1.equals(actualObj2); + } catch (JsonProcessingException e) { + throw new AtlasBaseException(JSON_ERROR, e.getMessage()); } } private void updateExistingVersion(EntityMutationContext context, AtlasEntity entity, AtlasEntity latestExistingVersion) throws AtlasBaseException { - context.getCreatedEntities().remove(entity); - Set guids = new HashSet<>(); - guids.add(entity.getGuid()); - entityStore.purgeByIds(guids); + removeCreatingVertex(context, entity); entity.setAttribute(QUALIFIED_NAME, latestExistingVersion.getAttribute(QUALIFIED_NAME)); entity.setGuid(latestExistingVersion.getGuid()); @@ -216,6 +201,19 @@ private void updateExistingVersion(EntityMutationContext context, AtlasEntity en } + private void removeCreatingVertex(EntityMutationContext context, AtlasEntity entity) throws AtlasBaseException { + context.getCreatedEntities().remove(entity); + try { + RequestContext.get().setSkipAuthorizationCheck(true); + Set guids = new HashSet<>(); + guids.add(entity.getGuid()); + entityStore.purgeByIds(guids); + } finally { + RequestContext.get().setSkipAuthorizationCheck(false); + } + + } + private void contractAttributeSync(AtlasEntity entity, DataContract contract) { // Sync certificateStatus if (!Objects.equals(entity.getAttribute(ATTR_CERTIFICATE_STATUS), contract.getStatus().name())) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java index e6d68f3895..a478466b9b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java @@ -1,13 +1,13 @@ package org.apache.atlas.repository.store.graph.v2.preprocessor.contract; import java.lang.String; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonProperty; -import com.fasterxml.jackson.annotation.JsonSetter; +import com.fasterxml.jackson.annotation.*; import com.fasterxml.jackson.core.JsonProcessingException; import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; @@ -16,9 +16,11 @@ import org.apache.commons.lang.StringUtils; import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; +import static org.apache.atlas.AtlasErrorCode.JSON_ERROR; @JsonIgnoreProperties(ignoreUnknown = true) +@JsonPropertyOrder({"kind", "status", "template_version", "dataset", "columns"}) public class DataContract { @JsonProperty(required = true) public String kind; @@ -27,7 +29,7 @@ public class DataContract { public String templateVersion; public Dataset dataset; public List columns; - + private Map unknownFields = new HashMap<>(); public STATUS getStatus() { return status; } @@ -37,6 +39,16 @@ public void setStatus(STATUS status) { this.status = status; } + @JsonAnySetter + public void setUnknownFields(String key, Object value) { + unknownFields.put(key, value); + } + + @JsonAnyGetter + public Map getUnknownFields() { + return unknownFields; + } + public enum STATUS { @JsonProperty("DRAFT") DRAFT, @JsonProperty("VERIFIED") VERIFIED; @@ -81,12 +93,23 @@ private boolean isSemVer(String version) { } @JsonIgnoreProperties(ignoreUnknown = true) + @JsonPropertyOrder({"name", "type", "description"}) public static final class Dataset { public String name; @JsonProperty(required = true) public DATASET_TYPE type; public String description; + private Map unknownFields = new HashMap<>(); + + @JsonAnySetter + public void setUnknownFields(String key, Object value) { + unknownFields.put(key, value); + } + @JsonAnyGetter + public Map getUnknownFields() { + return unknownFields; + } @JsonSetter("type") public void setType(DATASET_TYPE type) { @@ -117,6 +140,7 @@ public static DATASET_TYPE from(String s) throws AtlasBaseException { } @JsonIgnoreProperties(ignoreUnknown = true) + @JsonPropertyOrder({"name", "description", "data_type"}) public static final class Column { public String name; @@ -125,6 +149,17 @@ public static final class Column { public boolean is_primary; public String data_type; + private Map unknownFields = new HashMap<>(); + + @JsonAnySetter + public void setUnknownFields(String key, Object value) { + unknownFields.put(key, value); + } + @JsonAnyGetter + public Map getUnknownFields() { + return unknownFields; + } + } @@ -147,11 +182,15 @@ public static DataContract deserialize(String contractString) throws AtlasBaseEx return contract; } - public static String serialize(DataContract contract) throws JsonProcessingException { + public static String serialize(DataContract contract) throws AtlasBaseException { - ObjectMapper objectMapper = new ObjectMapper(); - objectMapper.enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS); - return objectMapper.writeValueAsString(contract); + try { + ObjectMapper objectMapper = new ObjectMapper(); + objectMapper.enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS); + return objectMapper.writeValueAsString(contract); + } catch (JsonProcessingException ex) { + throw new AtlasBaseException(JSON_ERROR, ex.getMessage()); + } } From d64e9690e9351fc0e6c5670262a2a1b27ee82da7 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Thu, 4 Apr 2024 12:10:16 +0530 Subject: [PATCH 04/25] Add attribute hasContract for assets --- .../contract/ContractPreProcessor.java | 31 +++++++++++-------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index adfe3e4e2a..38c8140008 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -35,6 +35,7 @@ public class ContractPreProcessor extends AbstractContractPreProcessor { public static final String ATTR_VERSION = "contractVersion"; public static final String ATTR_ASSET_QUALIFIED_NAME = "contractAssetQualifiedName"; public static final String ATTR_PARENT_GUID = "parentGuid"; + public static final String ATTR_HAS_CONTRACT = "hasContract"; public static final String CONTRACT_QUALIFIED_NAME_SUFFIX = "contract"; public static final String VERSION_PREFIX = "version"; public static final String CONTRACT_ATTR_STATUS = "status"; @@ -123,7 +124,7 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con updateExistingVersion(context, entity, latestExistingVersion); } // Contract status changed, either to draft or verified - return; + } else if (attributes.contains(ATTR_CONTRACT)) { //Contract is changed if (isEqualContract(contractString, (String) latestExistingVersion.getAttribute(ATTR_CONTRACT))) { @@ -138,8 +139,6 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con entity.setAttribute(ATTR_PARENT_GUID, latestExistingVersion.getGuid()); } - - return; } } else { @@ -149,11 +148,7 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con entity.setAttribute(ATTR_ASSET_QUALIFIED_NAME, associatedAsset.getEntity().getAttribute(QUALIFIED_NAME)); } - - if (contract.getStatus() == DataContract.STATUS.VERIFIED && - entity.getAttribute(ATTR_CERTIFICATE_STATUS).equals(DataContract.STATUS.VERIFIED.name())) { - datasetAttributeSync(associatedAsset.getEntity(), contract); - } + datasetAttributeSync(associatedAsset.getEntity(), contract, entity); } @@ -240,11 +235,21 @@ private void contractAttributeSync(AtlasEntity entity, DataContract contract) { } - private void datasetAttributeSync(AtlasEntity associatedAsset, DataContract contract) throws AtlasBaseException { - - DataContract.Dataset dataset = contract.dataset; - // Will implement dataset attribute sync from the contract attributes - + private void datasetAttributeSync(AtlasEntity associatedAsset, DataContract contract, AtlasEntity contractAsset) throws AtlasBaseException { + associatedAsset.setAttribute(ATTR_HAS_CONTRACT, true); + if (contract.getStatus() == DataContract.STATUS.VERIFIED && + contractAsset.getAttribute(ATTR_CERTIFICATE_STATUS).equals(DataContract.STATUS.VERIFIED.name())) { + DataContract.Dataset dataset = contract.dataset; + // Will implement dataset attribute sync from the contract attributes + } + try { + RequestContext.get().setSkipAuthorizationCheck(true); + EntityStream entityStream = new AtlasEntityStream(associatedAsset); + entityStore.createOrUpdate(entityStream, false); + LOG.info("Updated associated asset attributes of contract {}", associatedAsset.getAttribute(QUALIFIED_NAME)); + } finally { + RequestContext.get().setSkipAuthorizationCheck(false); + } } private static void validateAttribute(boolean isInvalid, String errorMessage) throws AtlasBaseException { From 6ee00aa7edf32c73e2c778b120589f0ab108ca24 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Tue, 9 Apr 2024 11:53:37 +0530 Subject: [PATCH 05/25] Add validation of data contract spec --- repository/pom.xml | 6 ++ .../contract/ContractPreProcessor.java | 36 +++++---- .../preprocessor/contract/DataContract.java | 79 +++++++++++++------ 3 files changed, 80 insertions(+), 41 deletions(-) diff --git a/repository/pom.xml b/repository/pom.xml index 10d8d876fb..64cb9c0d48 100755 --- a/repository/pom.xml +++ b/repository/pom.xml @@ -322,6 +322,12 @@ 3.0.0-SNAPSHOT + + org.hibernate + hibernate-validator + 4.3.0.Final + + diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index 38c8140008..fd82d9edd0 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -1,7 +1,6 @@ package org.apache.atlas.repository.store.graph.v2.preprocessor.contract; import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.core.type.TypeReference; import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.node.ObjectNode; @@ -10,7 +9,6 @@ import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; import org.apache.atlas.model.instance.AtlasStruct; -import org.apache.atlas.model.instance.EntityMutationResponse; import org.apache.atlas.model.instance.EntityMutations; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; @@ -35,7 +33,9 @@ public class ContractPreProcessor extends AbstractContractPreProcessor { public static final String ATTR_VERSION = "contractVersion"; public static final String ATTR_ASSET_QUALIFIED_NAME = "contractAssetQualifiedName"; public static final String ATTR_PARENT_GUID = "parentGuid"; - public static final String ATTR_HAS_CONTRACT = "hasContract"; + public static final String ASSET_ATTR_HAS_CONTRACT = "hasContract"; + public static final String ASSET_ATTR_DESCRIPTION = "description"; + public static final String CONTRACT_QUALIFIED_NAME_SUFFIX = "contract"; public static final String VERSION_PREFIX = "version"; public static final String CONTRACT_ATTR_STATUS = "status"; @@ -69,7 +69,7 @@ private void processUpdateContract(AtlasEntity entity, EntityMutationContext con String contractString = (String) entity.getAttribute(ATTR_CONTRACT); AtlasVertex vertex = context.getVertex(entity.getGuid()); AtlasEntity existingContractEntity = entityRetriever.toAtlasEntity(vertex); - + // TODO: Check for qualifiedName to understand if a particular version is getting updated or duplicate contract in payload if (!isEqualContract(contractString, (String) existingContractEntity.getAttribute(ATTR_CONTRACT))) { // Update the same asset(entity) throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Can't update a specific version of contract"); @@ -148,7 +148,7 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con entity.setAttribute(ATTR_ASSET_QUALIFIED_NAME, associatedAsset.getEntity().getAttribute(QUALIFIED_NAME)); } - datasetAttributeSync(associatedAsset.getEntity(), contract, entity); + datasetAttributeSync(context, associatedAsset.getEntity(), contract, entity); } @@ -209,7 +209,7 @@ private void removeCreatingVertex(EntityMutationContext context, AtlasEntity ent } - private void contractAttributeSync(AtlasEntity entity, DataContract contract) { + private void contractAttributeSync(AtlasEntity entity, DataContract contract) throws AtlasBaseException { // Sync certificateStatus if (!Objects.equals(entity.getAttribute(ATTR_CERTIFICATE_STATUS), contract.getStatus().name())) { /* @@ -223,32 +223,38 @@ private void contractAttributeSync(AtlasEntity entity, DataContract contract) { */ if (Objects.equals(entity.getAttribute(ATTR_CERTIFICATE_STATUS), DataContract.STATUS.VERIFIED.name())) { - contract.setStatus(DataContract.STATUS.VERIFIED); + contract.setStatus(String.valueOf(DataContract.STATUS.VERIFIED)); } else if (Objects.equals(contract.getStatus(), DataContract.STATUS.VERIFIED)) { entity.setAttribute(ATTR_CERTIFICATE_STATUS, DataContract.STATUS.VERIFIED.name()); } else { entity.setAttribute(ATTR_CERTIFICATE_STATUS, DataContract.STATUS.DRAFT); - contract.setStatus(DataContract.STATUS.DRAFT); + contract.setStatus(String.valueOf(DataContract.STATUS.DRAFT)); } } } - private void datasetAttributeSync(AtlasEntity associatedAsset, DataContract contract, AtlasEntity contractAsset) throws AtlasBaseException { - associatedAsset.setAttribute(ATTR_HAS_CONTRACT, true); + private void datasetAttributeSync(EntityMutationContext context, AtlasEntity associatedAsset, DataContract contract, AtlasEntity contractAsset) throws AtlasBaseException { + associatedAsset.setAttribute(ASSET_ATTR_HAS_CONTRACT, true); if (contract.getStatus() == DataContract.STATUS.VERIFIED && contractAsset.getAttribute(ATTR_CERTIFICATE_STATUS).equals(DataContract.STATUS.VERIFIED.name())) { DataContract.Dataset dataset = contract.dataset; // Will implement dataset attribute sync from the contract attributes + if (!dataset.description.isEmpty()) { + associatedAsset.setAttribute(ASSET_ATTR_DESCRIPTION, dataset.description); + } } try { - RequestContext.get().setSkipAuthorizationCheck(true); - EntityStream entityStream = new AtlasEntityStream(associatedAsset); - entityStore.createOrUpdate(entityStream, false); - LOG.info("Updated associated asset attributes of contract {}", associatedAsset.getAttribute(QUALIFIED_NAME)); + AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(associatedAsset.getGuid()); + AtlasEntityType entityType = ensureEntityType(associatedAsset.getTypeName()); + context.addUpdated(associatedAsset.getGuid(), associatedAsset, entityType, vertex); +// RequestContext.get().setSkipAuthorizationCheck(true); +// EntityStream entityStream = new AtlasEntityStream(associatedAsset); +// entityStore.createOrUpdate(entityStream, false); +// LOG.info("Updated associated asset attributes of contract {}", associatedAsset.getAttribute(QUALIFIED_NAME)); } finally { - RequestContext.get().setSkipAuthorizationCheck(false); +// RequestContext.get().setSkipAuthorizationCheck(false); } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java index a478466b9b..4e0887320c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java @@ -1,9 +1,7 @@ package org.apache.atlas.repository.store.graph.v2.preprocessor.contract; import java.lang.String; -import java.util.HashMap; -import java.util.List; -import java.util.Map; +import java.util.*; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -15,28 +13,39 @@ import org.apache.atlas.exception.AtlasBaseException; import org.apache.commons.lang.StringUtils; -import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; -import static org.apache.atlas.AtlasErrorCode.JSON_ERROR; +import javax.validation.*; +import javax.validation.constraints.NotNull; +import java.util.Set; + +import static org.apache.atlas.AtlasErrorCode.*; @JsonIgnoreProperties(ignoreUnknown = true) @JsonPropertyOrder({"kind", "status", "template_version", "dataset", "columns"}) public class DataContract { - @JsonProperty(required = true) - public String kind; - public STATUS status; + @Valid @NotNull + public String kind; + + public STATUS status; + @JsonProperty(value = "template_version", defaultValue = "0.0.1") - public String templateVersion; - public Dataset dataset; - public List columns; - private Map unknownFields = new HashMap<>(); + public String templateVersion; + @Valid @NotNull + public Dataset dataset; + @Valid + public List columns; + private Map unknownFields = new HashMap<>(); public STATUS getStatus() { return status; } @JsonSetter("status") - public void setStatus(STATUS status) { - this.status = status; + public void setStatus(String status) throws AtlasBaseException { + try { + this.status = STATUS.from(status); + } catch (IllegalArgumentException ex) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "status " + status + " is inappropriate. Accepted values: " + Arrays.toString(STATUS.values())); + } } @JsonAnySetter @@ -57,7 +66,6 @@ public static STATUS from(String s) { if(StringUtils.isEmpty(s)) { return DRAFT; } - switch (s.toLowerCase()) { case "draft": return DRAFT; @@ -79,9 +87,9 @@ public void setKind(String kind) throws AtlasBaseException { this.kind = kind; } - public void setTemplateVersion(String templateVersion) { + public void setTemplateVersion(String templateVersion) throws AtlasBaseException { if (!isSemVer(templateVersion)) { - throw new IllegalArgumentException("Invalid version syntax"); + throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "Invalid template_version syntax"); } this.templateVersion = templateVersion; } @@ -95,8 +103,9 @@ private boolean isSemVer(String version) { @JsonIgnoreProperties(ignoreUnknown = true) @JsonPropertyOrder({"name", "type", "description"}) public static final class Dataset { + @NotNull public String name; - @JsonProperty(required = true) + @NotNull public DATASET_TYPE type; public String description; private Map unknownFields = new HashMap<>(); @@ -112,8 +121,12 @@ public Map getUnknownFields() { } @JsonSetter("type") - public void setType(DATASET_TYPE type) { - this.type = type; + public void setType(String type) throws AtlasBaseException { + try { + this.type = DATASET_TYPE.from(type); + } catch (IllegalArgumentException | AtlasBaseException ex) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "dataset.type " + type + " is inappropriate. Accepted values: " + Arrays.toString(DATASET_TYPE.values())); + } } public enum DATASET_TYPE { @@ -131,7 +144,7 @@ public static DATASET_TYPE from(String s) throws AtlasBaseException { case "materialisedview": return MaterialisedView; default: - throw new AtlasBaseException("dataset.type value not supported yet."); + throw new AtlasBaseException(String.format("dataset.type: %s value not supported yet.", s)); } } } @@ -142,6 +155,7 @@ public static DATASET_TYPE from(String s) throws AtlasBaseException { @JsonIgnoreProperties(ignoreUnknown = true) @JsonPropertyOrder({"name", "description", "data_type"}) public static final class Column { + @NotNull public String name; public String description; @@ -160,8 +174,6 @@ public Map getUnknownFields() { return unknownFields; } - - } public static DataContract deserialize(String contractString) throws AtlasBaseException { @@ -175,11 +187,26 @@ public static DataContract deserialize(String contractString) throws AtlasBaseEx DataContract contract; try { contract = objectMapper.readValue(contractString, DataContract.class); - } catch (Exception ex) { - ex.printStackTrace(); - throw new AtlasBaseException("Failed at this"); + } catch (JsonProcessingException ex) { + throw new AtlasBaseException(ex.getOriginalMessage()); } + contract.validate(); return contract; + + } + + public void validate() throws AtlasBaseException { + Validator validator = Validation.buildDefaultValidatorFactory().getValidator(); + Set> violations = validator.validate(this); + if (!violations.isEmpty()) { + List errorMessageList = new ArrayList<>(); + for (ConstraintViolation violation : violations) { + errorMessageList.add(String.format("Field: %s -> %s", violation.getPropertyPath(), violation.getMessage())); + System.out.println(violation.getMessage()); + } + throw new AtlasBaseException(StringUtils.join(errorMessageList, "; ")); + } + } public static String serialize(DataContract contract) throws AtlasBaseException { From c79dfa065fe0fdadd9969f9a5d298eb9fa429986 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Tue, 16 Apr 2024 15:28:19 +0530 Subject: [PATCH 06/25] Fix audit logs for associated asset --- .../store/graph/v2/AtlasEntityStoreV2.java | 2 +- .../contract/ContractPreProcessor.java | 38 +++++++++++++++---- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 2b2c6a7a3c..ed24f329c6 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -1839,7 +1839,7 @@ public PreProcessor getPreProcessor(String typeName) { break; case CONTRACT_ENTITY_TYPE: - preProcessor = new ContractPreProcessor(graph, typeRegistry, entityRetriever, this, entityGraphMapper); + preProcessor = new ContractPreProcessor(graph, typeRegistry, entityRetriever, this, entityGraphMapper, storeDifferentialAudits); break; } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index fd82d9edd0..b64c151df7 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -34,6 +34,8 @@ public class ContractPreProcessor extends AbstractContractPreProcessor { public static final String ATTR_ASSET_QUALIFIED_NAME = "contractAssetQualifiedName"; public static final String ATTR_PARENT_GUID = "parentGuid"; public static final String ASSET_ATTR_HAS_CONTRACT = "hasContract"; + public static final String ASSET_ATTR_CONTRACT_VERSION_QUALIFIED_NAME = "latestContractQualifiedName"; + public static final String ASSET_ATTR_DESCRIPTION = "description"; public static final String CONTRACT_QUALIFIED_NAME_SUFFIX = "contract"; @@ -41,12 +43,14 @@ public class ContractPreProcessor extends AbstractContractPreProcessor { public static final String CONTRACT_ATTR_STATUS = "status"; private final AtlasEntityStore entityStore; private final EntityGraphMapper entityGraphMapper; + private boolean storeDifferentialAudits; public ContractPreProcessor(AtlasGraph graph, AtlasTypeRegistry typeRegistry, - EntityGraphRetriever entityRetriever, AtlasEntityStore entityStore, EntityGraphMapper entityGraphMapper) { + EntityGraphRetriever entityRetriever, AtlasEntityStore entityStore, EntityGraphMapper entityGraphMapper, boolean storeDifferentialAudits) { super(graph, typeRegistry, entityRetriever); + this.storeDifferentialAudits = storeDifferentialAudits; this.entityStore = entityStore; this.entityGraphMapper = entityGraphMapper; } @@ -236,19 +240,37 @@ private void contractAttributeSync(AtlasEntity entity, DataContract contract) th } private void datasetAttributeSync(EntityMutationContext context, AtlasEntity associatedAsset, DataContract contract, AtlasEntity contractAsset) throws AtlasBaseException { - associatedAsset.setAttribute(ASSET_ATTR_HAS_CONTRACT, true); + // Creating new empty AtlasEntity to update with selective attributes only + AtlasEntity entity = new AtlasEntity(associatedAsset.getTypeName()); + entity.setGuid(associatedAsset.getGuid()); + entity.setAttribute(QUALIFIED_NAME, associatedAsset.getAttribute(QUALIFIED_NAME)); + if (!associatedAsset.getAttribute(ASSET_ATTR_HAS_CONTRACT).equals(true)) { + entity.setAttribute(ASSET_ATTR_HAS_CONTRACT, true); + } + entity.setAttribute(ASSET_ATTR_CONTRACT_VERSION_QUALIFIED_NAME, contractAsset.getAttribute(QUALIFIED_NAME)); if (contract.getStatus() == DataContract.STATUS.VERIFIED && contractAsset.getAttribute(ATTR_CERTIFICATE_STATUS).equals(DataContract.STATUS.VERIFIED.name())) { DataContract.Dataset dataset = contract.dataset; // Will implement dataset attribute sync from the contract attributes - if (!dataset.description.isEmpty()) { - associatedAsset.setAttribute(ASSET_ATTR_DESCRIPTION, dataset.description); - } + // if (!dataset.description.isEmpty()) { + // associatedAsset.setAttribute(ASSET_ATTR_DESCRIPTION, dataset.description); + // } } try { - AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(associatedAsset.getGuid()); - AtlasEntityType entityType = ensureEntityType(associatedAsset.getTypeName()); - context.addUpdated(associatedAsset.getGuid(), associatedAsset, entityType, vertex); + AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(entity.getGuid()); + AtlasEntityType entityType = ensureEntityType(entity.getTypeName()); + AtlasEntityComparator entityComparator = new AtlasEntityComparator(typeRegistry, entityRetriever, context.getGuidAssignments(), true, true); + AtlasEntityComparator.AtlasEntityDiffResult diffResult = entityComparator.getDiffResult(entity, vertex, !storeDifferentialAudits); + RequestContext reqContext = RequestContext.get(); + + if (diffResult.hasDifference()) { + context.addUpdated(entity.getGuid(), entity, entityType, vertex); + if (storeDifferentialAudits) { + diffResult.getDiffEntity().setGuid(entity.getGuid()); + reqContext.cacheDifferentialEntity(diffResult.getDiffEntity()); + } + } + // RequestContext.get().setSkipAuthorizationCheck(true); // EntityStream entityStream = new AtlasEntityStream(associatedAsset); // entityStore.createOrUpdate(entityStream, false); From 723286a517402ed120445e1634bff3fe95e0ff39 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Tue, 16 Apr 2024 17:43:55 +0530 Subject: [PATCH 07/25] Update dataset spec structure --- .../contract/ContractPreProcessor.java | 4 +- .../preprocessor/contract/DataContract.java | 87 +++++++------------ 2 files changed, 34 insertions(+), 57 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index b64c151df7..c4e75baf46 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -99,8 +99,7 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con String contractString = (String) entity.getAttribute(ATTR_CONTRACT); DataContract contract = DataContract.deserialize(contractString); - DataContract.Dataset dataset = contract.dataset; - AtlasEntityWithExtInfo associatedAsset = getAssociatedAsset(contractQName, dataset.type.name()); + AtlasEntityWithExtInfo associatedAsset = getAssociatedAsset(contractQName, contract.type.name()); authorizeContractCreateOrUpdate(entity, associatedAsset); @@ -250,7 +249,6 @@ private void datasetAttributeSync(EntityMutationContext context, AtlasEntity ass entity.setAttribute(ASSET_ATTR_CONTRACT_VERSION_QUALIFIED_NAME, contractAsset.getAttribute(QUALIFIED_NAME)); if (contract.getStatus() == DataContract.STATUS.VERIFIED && contractAsset.getAttribute(ATTR_CERTIFICATE_STATUS).equals(DataContract.STATUS.VERIFIED.name())) { - DataContract.Dataset dataset = contract.dataset; // Will implement dataset attribute sync from the contract attributes // if (!dataset.description.isEmpty()) { // associatedAsset.setAttribute(ASSET_ATTR_DESCRIPTION, dataset.description); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java index 4e0887320c..18413f06cd 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java @@ -21,7 +21,7 @@ @JsonIgnoreProperties(ignoreUnknown = true) -@JsonPropertyOrder({"kind", "status", "template_version", "dataset", "columns"}) +@JsonPropertyOrder({"kind", "status", "template_version", "dataset", "type", "columns"}) public class DataContract { @Valid @NotNull public String kind; @@ -31,10 +31,41 @@ public class DataContract { @JsonProperty(value = "template_version", defaultValue = "0.0.1") public String templateVersion; @Valid @NotNull - public Dataset dataset; + public String dataset; + @Valid @NotNull + public DATASET_TYPE type; @Valid public List columns; private Map unknownFields = new HashMap<>(); + + @JsonSetter("type") + public void setType(String type) throws AtlasBaseException { + try { + this.type = DATASET_TYPE.from(type); + } catch (IllegalArgumentException | AtlasBaseException ex) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "type " + type + " is inappropriate. Accepted values: " + Arrays.toString(DATASET_TYPE.values())); + } + } + + public enum DATASET_TYPE { + @JsonProperty("Table") Table, + @JsonProperty("View") View, + @JsonProperty("MaterialisedView") MaterialisedView; + + public static DATASET_TYPE from(String s) throws AtlasBaseException { + + switch (s.toLowerCase()) { + case "table": + return Table; + case "view": + return View; + case "materialisedview": + return MaterialisedView; + default: + throw new AtlasBaseException(String.format("dataset.type: %s value not supported yet.", s)); + } + } + } public STATUS getStatus() { return status; } @@ -100,58 +131,6 @@ private boolean isSemVer(String version) { return matcher.matches(); } - @JsonIgnoreProperties(ignoreUnknown = true) - @JsonPropertyOrder({"name", "type", "description"}) - public static final class Dataset { - @NotNull - public String name; - @NotNull - public DATASET_TYPE type; - public String description; - private Map unknownFields = new HashMap<>(); - - @JsonAnySetter - public void setUnknownFields(String key, Object value) { - unknownFields.put(key, value); - } - - @JsonAnyGetter - public Map getUnknownFields() { - return unknownFields; - } - - @JsonSetter("type") - public void setType(String type) throws AtlasBaseException { - try { - this.type = DATASET_TYPE.from(type); - } catch (IllegalArgumentException | AtlasBaseException ex) { - throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "dataset.type " + type + " is inappropriate. Accepted values: " + Arrays.toString(DATASET_TYPE.values())); - } - } - - public enum DATASET_TYPE { - @JsonProperty("Table") Table, - @JsonProperty("View") View, - @JsonProperty("MaterialisedView") MaterialisedView; - - public static DATASET_TYPE from(String s) throws AtlasBaseException { - - switch (s.toLowerCase()) { - case "table": - return Table; - case "view": - return View; - case "materialisedview": - return MaterialisedView; - default: - throw new AtlasBaseException(String.format("dataset.type: %s value not supported yet.", s)); - } - } - } - - - } - @JsonIgnoreProperties(ignoreUnknown = true) @JsonPropertyOrder({"name", "description", "data_type"}) public static final class Column { From 3e6f1b2931c54910cf80d443dfc6b0605bd6018f Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Tue, 16 Apr 2024 18:07:43 +0530 Subject: [PATCH 08/25] Minor bug fix --- .../graph/v2/preprocessor/contract/ContractPreProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index c4e75baf46..e7c165a328 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -243,7 +243,7 @@ private void datasetAttributeSync(EntityMutationContext context, AtlasEntity ass AtlasEntity entity = new AtlasEntity(associatedAsset.getTypeName()); entity.setGuid(associatedAsset.getGuid()); entity.setAttribute(QUALIFIED_NAME, associatedAsset.getAttribute(QUALIFIED_NAME)); - if (!associatedAsset.getAttribute(ASSET_ATTR_HAS_CONTRACT).equals(true)) { + if (associatedAsset.getAttribute(ASSET_ATTR_HAS_CONTRACT) == null || associatedAsset.getAttribute(ASSET_ATTR_HAS_CONTRACT).equals(false)) { entity.setAttribute(ASSET_ATTR_HAS_CONTRACT, true); } entity.setAttribute(ASSET_ATTR_CONTRACT_VERSION_QUALIFIED_NAME, contractAsset.getAttribute(QUALIFIED_NAME)); From f69d6f688295666c83fc2afdf73b48ae3f1fd41c Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Mon, 22 Apr 2024 17:41:44 +0530 Subject: [PATCH 09/25] Refactor with new attributes --- .../store/graph/v2/AtlasEntityStoreV2.java | 2 +- .../AbstractContractPreProcessor.java | 16 --- .../contract/ContractPreProcessor.java | 129 +++++++----------- .../contract/ContractVersionUtils.java | 113 ++++----------- 4 files changed, 82 insertions(+), 178 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index ed24f329c6..8e11aaeff1 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -1839,7 +1839,7 @@ public PreProcessor getPreProcessor(String typeName) { break; case CONTRACT_ENTITY_TYPE: - preProcessor = new ContractPreProcessor(graph, typeRegistry, entityRetriever, this, entityGraphMapper, storeDifferentialAudits); + preProcessor = new ContractPreProcessor(graph, typeRegistry, entityRetriever, this, storeDifferentialAudits, discovery); break; } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java index dd2e558156..4ef8d0ee6d 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java @@ -55,22 +55,6 @@ void authorizeContractCreateOrUpdate(AtlasEntity contractEntity, AtlasEntity.Atl } - void authorizeContractDelete(AtlasVertex contractVertex, String typeName) throws AtlasBaseException { - AtlasPerfMetrics.MetricRecorder recorder = RequestContext.get().startMetricRecord("authorizeContractDelete"); - - try { - AtlasEntity contractEntity = entityRetriever.toAtlasEntity(contractVertex); - String contractQName = contractEntity.getAttribute(QUALIFIED_NAME).toString(); - AtlasEntity.AtlasEntityWithExtInfo assetEntity = getAssociatedAsset(contractQName, typeName); - AtlasEntityHeader entityHeader = new AtlasEntityHeader(assetEntity.getEntity()); - - verifyAssetAccess(entityHeader, AtlasPrivilege.ENTITY_UPDATE, contractEntity, AtlasPrivilege.ENTITY_DELETE); - } finally { - RequestContext.get().endMetricRecord(recorder); - } - } - - private void verifyAssetAccess(AtlasEntityHeader asset, AtlasPrivilege assetPrivilege, AtlasEntity contract, AtlasPrivilege contractPrivilege) throws AtlasBaseException { verifyAccess(asset, assetPrivilege); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index e7c165a328..173f066753 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -5,6 +5,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode; import org.apache.atlas.RequestContext; +import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; @@ -26,33 +27,36 @@ import static org.apache.atlas.AtlasErrorCode.*; import static org.apache.atlas.repository.Constants.ATTR_CERTIFICATE_STATUS; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; +import static org.apache.atlas.type.AtlasTypeUtil.getAtlasObjectId; public class ContractPreProcessor extends AbstractContractPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(ContractPreProcessor.class); - public static final String ATTR_CONTRACT = "contract"; - public static final String ATTR_VERSION = "contractVersion"; - public static final String ATTR_ASSET_QUALIFIED_NAME = "contractAssetQualifiedName"; - public static final String ATTR_PARENT_GUID = "parentGuid"; + public static final String ATTR_CONTRACT = "dataContractJson"; + public static final String ATTR_VERSION = "dataContractVersion"; + public static final String REL_ATTR_GOVERNED_ASSET = "dataContractGovernedAsset"; + public static final String REL_ATTR_LATEST_CONTRACT = "dataContractLatest"; + public static final String REL_ATTR_GOVERNED_ASSET_CERTIFIED = "dataContractGovernedAssetCertified"; + public static final String REL_ATTR_PREVIOUS_VERSION = "dataContractPreviousVersion"; public static final String ASSET_ATTR_HAS_CONTRACT = "hasContract"; - public static final String ASSET_ATTR_CONTRACT_VERSION_QUALIFIED_NAME = "latestContractQualifiedName"; - public static final String ASSET_ATTR_DESCRIPTION = "description"; public static final String CONTRACT_QUALIFIED_NAME_SUFFIX = "contract"; public static final String VERSION_PREFIX = "version"; public static final String CONTRACT_ATTR_STATUS = "status"; private final AtlasEntityStore entityStore; - private final EntityGraphMapper entityGraphMapper; - private boolean storeDifferentialAudits; + private final boolean storeDifferentialAudits; + private EntityDiscoveryService discovery; + public ContractPreProcessor(AtlasGraph graph, AtlasTypeRegistry typeRegistry, - EntityGraphRetriever entityRetriever, AtlasEntityStore entityStore, EntityGraphMapper entityGraphMapper, boolean storeDifferentialAudits) { + EntityGraphRetriever entityRetriever, AtlasEntityStore entityStore, + boolean storeDifferentialAudits, EntityDiscoveryService discovery) { super(graph, typeRegistry, entityRetriever); this.storeDifferentialAudits = storeDifferentialAudits; this.entityStore = entityStore; - this.entityGraphMapper = entityGraphMapper; + this.discovery = discovery; } @Override @@ -107,50 +111,38 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con contractString = DataContract.serialize(contract); entity.setAttribute(ATTR_CONTRACT, contractString); - - ContractVersionUtils versionUtil = new ContractVersionUtils(entity, context, entityRetriever, typeRegistry, entityStore, graph); - AtlasEntity latestExistingVersion = versionUtil.getLatestVersion(); - - if (latestExistingVersion != null) { + ContractVersionUtils versionUtil = new ContractVersionUtils(entity, context, entityRetriever, typeRegistry, entityStore, graph, discovery); + AtlasEntity currentVersionEntity = versionUtil.getCurrentVersion(); + String latestVersion = "V1"; + if (currentVersionEntity != null) { // Contract already exist - String qName = (String) latestExistingVersion.getAttribute(QUALIFIED_NAME); - Integer latestVersionNumber = Integer.valueOf(qName.substring(qName.lastIndexOf("/V") + 2)); - List attributes = getDiffAttributes(context, entity, latestExistingVersion); + String qName = (String) currentVersionEntity.getAttribute(QUALIFIED_NAME); + Integer currentVersionNumber = Integer.valueOf(qName.substring(qName.lastIndexOf("/V") + 2)); + List attributes = getDiffAttributes(context, entity, currentVersionEntity); if (attributes.isEmpty()) { + // No changes in the contract, Not creating new version removeCreatingVertex(context, entity); return; - } - - if (attributes.size() == 1 && attributes.contains(ATTR_CERTIFICATE_STATUS)) { - if (Objects.equals(entity.getAttribute(ATTR_CERTIFICATE_STATUS).toString(), DataContract.STATUS.VERIFIED.name())) { - //update existing entity - updateExistingVersion(context, entity, latestExistingVersion); - } - // Contract status changed, either to draft or verified - - } else if (attributes.contains(ATTR_CONTRACT)) { - //Contract is changed - if (isEqualContract(contractString, (String) latestExistingVersion.getAttribute(ATTR_CONTRACT))) { - // Update the same asset(entity) - updateExistingVersion(context, entity, latestExistingVersion); + } else if (isEqualContract(contractString, (String) currentVersionEntity.getAttribute(ATTR_CONTRACT))) { + // No change in contract, metadata changed + updateExistingVersion(context, entity, currentVersionEntity); + return; + } else { + // contract changed (metadata might/not changed). Create new version. + latestVersion = String.format("V%s", currentVersionNumber + 1); - } else { - // Create New version of entity - entity.setAttribute(QUALIFIED_NAME, String.format("%s/%s/V%s", contractQName, VERSION_PREFIX, ++latestVersionNumber)); - entity.setAttribute(ATTR_VERSION, String.format("V%s", latestVersionNumber)); - entity.setAttribute(ATTR_ASSET_QUALIFIED_NAME, associatedAsset.getEntity().getAttribute(QUALIFIED_NAME)); - entity.setAttribute(ATTR_PARENT_GUID, latestExistingVersion.getGuid()); + // Attach previous version via rel + entity.setRelationshipAttribute(REL_ATTR_PREVIOUS_VERSION, getAtlasObjectId(currentVersionEntity)); + AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(currentVersionEntity.getGuid()); + AtlasEntityType entityType = ensureEntityType(currentVersionEntity.getTypeName()); + context.addUpdated(currentVersionEntity.getGuid(), currentVersionEntity, entityType, vertex); - } } - - } else { - // Create new contract - entity.setAttribute(QUALIFIED_NAME, String.format("%s/%s/%s", contractQName, VERSION_PREFIX, "V1")); - entity.setAttribute(ATTR_VERSION, "V1"); - entity.setAttribute(ATTR_ASSET_QUALIFIED_NAME, associatedAsset.getEntity().getAttribute(QUALIFIED_NAME)); - } + entity.setAttribute(QUALIFIED_NAME, String.format("%s/%s/%s", contractQName, VERSION_PREFIX, latestVersion)); + entity.setAttribute(ATTR_VERSION, latestVersion); + entity.setRelationshipAttribute(REL_ATTR_GOVERNED_ASSET, getAtlasObjectId(associatedAsset.getEntity())); + datasetAttributeSync(context, associatedAsset.getEntity(), contract, entity); } @@ -186,10 +178,10 @@ private boolean isEqualContract(String firstNode, String secondNode) throws Atla } - private void updateExistingVersion(EntityMutationContext context, AtlasEntity entity, AtlasEntity latestExistingVersion) throws AtlasBaseException { + private void updateExistingVersion(EntityMutationContext context, AtlasEntity entity, AtlasEntity currentVersionEntity) throws AtlasBaseException { removeCreatingVertex(context, entity); - entity.setAttribute(QUALIFIED_NAME, latestExistingVersion.getAttribute(QUALIFIED_NAME)); - entity.setGuid(latestExistingVersion.getGuid()); + entity.setAttribute(QUALIFIED_NAME, currentVersionEntity.getAttribute(QUALIFIED_NAME)); + entity.setGuid(currentVersionEntity.getGuid()); AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(entity.getGuid()); @@ -246,35 +238,18 @@ private void datasetAttributeSync(EntityMutationContext context, AtlasEntity ass if (associatedAsset.getAttribute(ASSET_ATTR_HAS_CONTRACT) == null || associatedAsset.getAttribute(ASSET_ATTR_HAS_CONTRACT).equals(false)) { entity.setAttribute(ASSET_ATTR_HAS_CONTRACT, true); } - entity.setAttribute(ASSET_ATTR_CONTRACT_VERSION_QUALIFIED_NAME, contractAsset.getAttribute(QUALIFIED_NAME)); - if (contract.getStatus() == DataContract.STATUS.VERIFIED && - contractAsset.getAttribute(ATTR_CERTIFICATE_STATUS).equals(DataContract.STATUS.VERIFIED.name())) { - // Will implement dataset attribute sync from the contract attributes - // if (!dataset.description.isEmpty()) { - // associatedAsset.setAttribute(ASSET_ATTR_DESCRIPTION, dataset.description); - // } - } - try { - AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(entity.getGuid()); - AtlasEntityType entityType = ensureEntityType(entity.getTypeName()); - AtlasEntityComparator entityComparator = new AtlasEntityComparator(typeRegistry, entityRetriever, context.getGuidAssignments(), true, true); - AtlasEntityComparator.AtlasEntityDiffResult diffResult = entityComparator.getDiffResult(entity, vertex, !storeDifferentialAudits); - RequestContext reqContext = RequestContext.get(); - - if (diffResult.hasDifference()) { - context.addUpdated(entity.getGuid(), entity, entityType, vertex); - if (storeDifferentialAudits) { - diffResult.getDiffEntity().setGuid(entity.getGuid()); - reqContext.cacheDifferentialEntity(diffResult.getDiffEntity()); - } - } + AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(entity.getGuid()); + AtlasEntityType entityType = ensureEntityType(entity.getTypeName()); + AtlasEntityComparator entityComparator = new AtlasEntityComparator(typeRegistry, entityRetriever, context.getGuidAssignments(), true, true); + AtlasEntityComparator.AtlasEntityDiffResult diffResult = entityComparator.getDiffResult(entity, vertex, !storeDifferentialAudits); + RequestContext reqContext = RequestContext.get(); + context.addUpdated(entity.getGuid(), entity, entityType, vertex); -// RequestContext.get().setSkipAuthorizationCheck(true); -// EntityStream entityStream = new AtlasEntityStream(associatedAsset); -// entityStore.createOrUpdate(entityStream, false); -// LOG.info("Updated associated asset attributes of contract {}", associatedAsset.getAttribute(QUALIFIED_NAME)); - } finally { -// RequestContext.get().setSkipAuthorizationCheck(false); + if (diffResult.hasDifference()) { + if (storeDifferentialAudits) { + diffResult.getDiffEntity().setGuid(entity.getGuid()); + reqContext.cacheDifferentialEntity(diffResult.getDiffEntity()); + } } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java index 8d27a10bc1..3f93d305e9 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java @@ -1,29 +1,23 @@ package org.apache.atlas.repository.store.graph.v2.preprocessor.contract; -import org.apache.atlas.AtlasErrorCode; -import org.apache.atlas.RequestContext; +import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.discovery.AtlasSearchResult; +import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.repository.graphdb.AtlasGraph; -import org.apache.atlas.repository.graphdb.AtlasGraphQuery; -import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.AtlasEntityStore; import org.apache.atlas.repository.store.graph.v2.*; -import org.apache.atlas.repository.store.graph.v2.preprocessor.ConnectionPreProcessor; -import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; -import org.apache.atlas.utils.AtlasPerfMetrics; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; import java.util.*; import static org.apache.atlas.repository.Constants.*; -import static org.apache.atlas.repository.graph.AtlasGraphProvider.getGraphInstance; +import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; public class ContractVersionUtils { - private static final Logger LOG = LoggerFactory.getLogger(ConnectionPreProcessor.class); private final EntityMutationContext context; public final EntityGraphRetriever entityRetriever; @@ -33,58 +27,46 @@ public class ContractVersionUtils { private AtlasEntity entity; public final AtlasGraph graph; - private List versionList; - - + private List versionList; + private EntityDiscoveryService discovery; public ContractVersionUtils(AtlasEntity entity, EntityMutationContext context, EntityGraphRetriever entityRetriever, - AtlasTypeRegistry atlasTypeRegistry, AtlasEntityStore entityStore, AtlasGraph graph) { + AtlasTypeRegistry atlasTypeRegistry, AtlasEntityStore entityStore, AtlasGraph graph, + EntityDiscoveryService discovery) { this.context = context; this.entityRetriever = entityRetriever; this.atlasTypeRegistry = atlasTypeRegistry; this.graph = graph; this.entityStore = entityStore; this.entity = entity; + this.discovery = discovery; } - private void extractAllVersions() { + private void extractAllVersions() throws AtlasBaseException { String contractQName = (String) entity.getAttribute(QUALIFIED_NAME); String datasetQName = contractQName.substring(0, contractQName.lastIndexOf("/contract")); + List ret = new ArrayList<>(); - AtlasEntityType entityType = atlasTypeRegistry.getEntityTypeByName("DataContract"); - Integer versionCounter = 1; - boolean found = true; - - List versionList = new ArrayList<>(); - - while (found) { - Map uniqAttributes = new HashMap<>(); - uniqAttributes.put(QUALIFIED_NAME, String.format("%s/contract/version/V%s", datasetQName, versionCounter++)); - try { - AtlasVertex entityVertex = AtlasGraphUtilsV2.getVertexByUniqueAttributes(graph, entityType, uniqAttributes); - AtlasEntity entity = entityRetriever.toAtlasEntity(entityVertex); -// -// EntityGraphRetriever entityRetriever = new EntityGraphRetriever(graph, typeRegistry, true); -// -// AtlasEntity.AtlasEntityWithExtInfo ret = entityRetriever.toAtlasEntityWithExtInfo(entityVertex); -// -// if (ret == null) { -// throw new AtlasBaseException(AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND, entityType.getTypeName(), -// uniqAttributes.toString()); -// } -// return ret; - - versionList.add(entity); - } catch (AtlasBaseException ex) { - found = false; - } + IndexSearchParams indexSearchParams = new IndexSearchParams(); + Map dsl = new HashMap<>(); - } - this.versionList = versionList; + List mustClauseList = new ArrayList(); + mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", CONTRACT_ENTITY_TYPE))); + mustClauseList.add(mapOf("wildcard", mapOf(QUALIFIED_NAME, String.format("%s/contract/version/*", datasetQName)))); + + dsl.put("query", mapOf("bool", mapOf("must", mustClauseList))); + indexSearchParams.setDsl(dsl); + indexSearchParams.setSuppressLogs(true); + + AtlasSearchResult result = discovery.directIndexSearch(indexSearchParams); + if (result != null) { + ret = result.getEntities(); + } + this.versionList = ret; } - public AtlasEntity getLatestVersion() throws AtlasBaseException { + public AtlasEntity getCurrentVersion() throws AtlasBaseException { if (this.versionList == null) { extractAllVersions(); } @@ -97,43 +79,6 @@ public AtlasEntity getLatestVersion() throws AtlasBaseException { if (this.versionList.isEmpty()) { return null; } - return this.versionList.get(0); - } - - - public Iterator getAllEntityVersions() { - String entityQNamePrefix = (String) entity.getAttribute(QUALIFIED_NAME); - AtlasEntityType entityType = atlasTypeRegistry.getEntityTypeByName("DataContract"); - -// AtlasEntityType entityType, String name - AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("getAllEntityVersions"); - AtlasGraph graph = getGraphInstance(); - AtlasGraphQuery query = graph.query() - .has(ENTITY_TYPE_PROPERTY_KEY, entityType.getTypeName()) - .has(STATE_PROPERTY_KEY, AtlasEntity.Status.ACTIVE.name()) - .has(entityType.getAllAttributes().get(QUALIFIED_NAME).getQualifiedName(), String.format("%s/version/V1", entityQNamePrefix)); - - - Iterator result = query.vertices().iterator(); - - RequestContext.get().endMetricRecord(metric); - return result; + return new AtlasEntity(this.versionList.get(0)); } - -// public void createNewVersion() throws AtlasBaseException { -// AtlasVertex vertex = context.getVertex(entity.getGuid()); -// AtlasEntity existingContractEntity = entityRetriever.toAtlasEntity(vertex); -//// this.newEntity = new AtlasEntity(existingEntity); -// this.existingEntity.setAttribute(QUALIFIED_NAME, null); -// -// try { -// RequestContext.get().setSkipAuthorizationCheck(true); -// EntityStream entityStream = new AtlasEntityStream(existingEntity); -// entityStore.createOrUpdate(entityStream, false); -// LOG.info("Created bootstrap policies for connection {}", existingEntity.getAttribute(QUALIFIED_NAME)); -// } finally { -// RequestContext.get().setSkipAuthorizationCheck(false); -// } -// -// } } From 3e75d0f3af1894179ac7006d5048197e2f278487 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Tue, 23 Apr 2024 15:17:12 +0530 Subject: [PATCH 10/25] Fix order of attributes in contract and change dataContractVersion to integer --- .../contract/ContractPreProcessor.java | 18 ++++++----- .../contract/ContractVersionUtils.java | 6 ++-- .../preprocessor/contract/DataContract.java | 30 +++++++++++++++++-- 3 files changed, 40 insertions(+), 14 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index 173f066753..5bf59414cb 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -33,9 +33,8 @@ public class ContractPreProcessor extends AbstractContractPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(ContractPreProcessor.class); public static final String ATTR_CONTRACT = "dataContractJson"; public static final String ATTR_VERSION = "dataContractVersion"; - public static final String REL_ATTR_GOVERNED_ASSET = "dataContractGovernedAsset"; - public static final String REL_ATTR_LATEST_CONTRACT = "dataContractLatest"; - public static final String REL_ATTR_GOVERNED_ASSET_CERTIFIED = "dataContractGovernedAssetCertified"; + public static final String REL_ATTR_GOVERNED_ASSET = "dataContractAsset"; + public static final String REL_ATTR_GOVERNED_ASSET_CERTIFIED = "dataContractAssetCertified"; public static final String REL_ATTR_PREVIOUS_VERSION = "dataContractPreviousVersion"; public static final String ASSET_ATTR_HAS_CONTRACT = "hasContract"; public static final String ASSET_ATTR_DESCRIPTION = "description"; @@ -113,11 +112,11 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con ContractVersionUtils versionUtil = new ContractVersionUtils(entity, context, entityRetriever, typeRegistry, entityStore, graph, discovery); AtlasEntity currentVersionEntity = versionUtil.getCurrentVersion(); - String latestVersion = "V1"; + int newVersionNumber = 1; if (currentVersionEntity != null) { // Contract already exist String qName = (String) currentVersionEntity.getAttribute(QUALIFIED_NAME); - Integer currentVersionNumber = Integer.valueOf(qName.substring(qName.lastIndexOf("/V") + 2)); + int currentVersionNumber = Integer.parseInt(qName.substring(qName.lastIndexOf("/V") + 2)); List attributes = getDiffAttributes(context, entity, currentVersionEntity); if (attributes.isEmpty()) { // No changes in the contract, Not creating new version @@ -129,7 +128,7 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con return; } else { // contract changed (metadata might/not changed). Create new version. - latestVersion = String.format("V%s", currentVersionNumber + 1); + newVersionNumber = currentVersionNumber + 1; // Attach previous version via rel entity.setRelationshipAttribute(REL_ATTR_PREVIOUS_VERSION, getAtlasObjectId(currentVersionEntity)); @@ -139,9 +138,12 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con } } - entity.setAttribute(QUALIFIED_NAME, String.format("%s/%s/%s", contractQName, VERSION_PREFIX, latestVersion)); - entity.setAttribute(ATTR_VERSION, latestVersion); + entity.setAttribute(QUALIFIED_NAME, String.format("%s/%s/V%s", contractQName, VERSION_PREFIX, newVersionNumber)); + entity.setAttribute(ATTR_VERSION, newVersionNumber); entity.setRelationshipAttribute(REL_ATTR_GOVERNED_ASSET, getAtlasObjectId(associatedAsset.getEntity())); + if (Objects.equals(entity.getAttribute(ATTR_CERTIFICATE_STATUS), DataContract.STATUS.VERIFIED.name()) ) { + entity.setRelationshipAttribute(REL_ATTR_GOVERNED_ASSET_CERTIFIED, getAtlasObjectId(associatedAsset.getEntity())); + } datasetAttributeSync(context, associatedAsset.getEntity(), contract, entity); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java index 3f93d305e9..e803e51b1f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java @@ -70,15 +70,15 @@ public AtlasEntity getCurrentVersion() throws AtlasBaseException { if (this.versionList == null) { extractAllVersions(); } + if (this.versionList == null) { + return null; + } Collections.sort(this.versionList, (e1, e2) -> { String e1QName = (String) e1.getAttribute(QUALIFIED_NAME); String e2QName = (String) e2.getAttribute(QUALIFIED_NAME); return e2QName.compareTo(e1QName); }); - if (this.versionList.isEmpty()) { - return null; - } return new AtlasEntity(this.versionList.get(0)); } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java index 18413f06cd..d96c885c4f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java @@ -21,7 +21,8 @@ @JsonIgnoreProperties(ignoreUnknown = true) -@JsonPropertyOrder({"kind", "status", "template_version", "dataset", "type", "columns"}) +@JsonPropertyOrder({"kind", "status", "template_version", "datasource", "dataset", "type", "description", "owners", + "tags", "certificate", "columns"}) public class DataContract { @Valid @NotNull public String kind; @@ -31,12 +32,18 @@ public class DataContract { @JsonProperty(value = "template_version", defaultValue = "0.0.1") public String templateVersion; @Valid @NotNull + public String datasource; + @Valid @NotNull public String dataset; @Valid @NotNull public DATASET_TYPE type; + public String description; + public List owners; + public List tags; + public String certificate; @Valid - public List columns; - private Map unknownFields = new HashMap<>(); + public List columns; + private Map unknownFields = new HashMap<>(); @JsonSetter("type") public void setType(String type) throws AtlasBaseException { @@ -131,6 +138,23 @@ private boolean isSemVer(String version) { return matcher.matches(); } + @JsonIgnoreProperties(ignoreUnknown = true) + @JsonPropertyOrder({"name"}) + public static final class BusinessTag { + @NotNull + public String name; + private Map unknownFields = new HashMap<>(); + + @JsonAnySetter + public void setUnknownFields(String key, Object value) { + unknownFields.put(key, value); + } + @JsonAnyGetter + public Map getUnknownFields() { + return unknownFields; + } + + } @JsonIgnoreProperties(ignoreUnknown = true) @JsonPropertyOrder({"name", "description", "data_type"}) public static final class Column { From 2e30f63bca2b291e144ee702c0d734e4aee60664 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Tue, 23 Apr 2024 17:45:58 +0530 Subject: [PATCH 11/25] Add bootstrap policy for DataContract CRU --- .../policies/bootstrap_entity_policies.json | 102 ++++++++++++++++++ 1 file changed, 102 insertions(+) diff --git a/addons/policies/bootstrap_entity_policies.json b/addons/policies/bootstrap_entity_policies.json index a889e0ef34..ac8629faeb 100644 --- a/addons/policies/bootstrap_entity_policies.json +++ b/addons/policies/bootstrap_entity_policies.json @@ -3004,6 +3004,108 @@ "entity-delete" ] } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "CREATE_DATA_CONTRACT", + "qualifiedName": "CREATE_DATA_CONTRACT", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:DataContract", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-create" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "READ_DATA_CONTRACT", + "qualifiedName": "READ_DATA_CONTRACT", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:DataContract", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-read" + ] + } + }, + { + "typeName": "AuthPolicy", + "attributes": + { + "name": "UPDATE_DATA_CONTRACT", + "qualifiedName": "UPDATE_DATA_CONTRACT", + "policyCategory": "bootstrap", + "policySubCategory": "default", + "policyServiceName": "atlas", + "policyType": "allow", + "policyPriority": 1, + "policyUsers": + [], + "policyGroups": + [], + "policyRoles": + [ + "$admin", + "$member", + "$api-token-default-access" + ], + "policyResourceCategory": "ENTITY", + "policyResources": + [ + "entity-type:DataContract", + "entity-classification:*", + "entity:*" + ], + "policyActions": + [ + "entity-update" + ] + } } ] } From 32104e90d347c416902c22a9450150fc8ccec2f6 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Wed, 24 Apr 2024 10:33:07 +0530 Subject: [PATCH 12/25] Fix qualifiedName of the contract --- .../org/apache/atlas/repository/Constants.java | 2 +- .../contract/AbstractContractPreProcessor.java | 3 +-- .../contract/ContractPreProcessor.java | 10 ++++++---- .../contract/ContractVersionUtils.java | 14 +++++++------- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 925d0a6476..2152921981 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -400,7 +400,7 @@ public enum SupportedFileExtensions { XLSX, XLS, CSV } public static final String ATTR_ASSET_STARRED_BY = "assetStarredBy"; public static final String ATTR_ASSET_STARRED_AT = "assetStarredAt"; public static final String ATTR_CERTIFICATE_STATUS = "certificateStatus"; - + public static final String ATTR_CONTRACT = "dataContractJson"; public static final String STRUCT_STARRED_DETAILS = "StarredDetails"; public static final String KEYCLOAK_ROLE_ADMIN = "$admin"; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java index 4ef8d0ee6d..9b063d38f5 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java @@ -70,8 +70,7 @@ private void verifyAccess(AtlasEntityHeader entityHeader, AtlasPrivilege privile AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, privilege, entityHeader), errorMessage); } - AtlasEntity.AtlasEntityWithExtInfo getAssociatedAsset(String contractQName, String typeName) throws AtlasBaseException { - String datasetQName = contractQName.substring(0, contractQName.lastIndexOf('/')); + AtlasEntity.AtlasEntityWithExtInfo getAssociatedAsset(String datasetQName, String typeName) throws AtlasBaseException { Map uniqAttributes = new HashMap<>(); uniqAttributes.put(QUALIFIED_NAME, datasetQName); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index 5bf59414cb..b1c2474d41 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -27,11 +27,11 @@ import static org.apache.atlas.AtlasErrorCode.*; import static org.apache.atlas.repository.Constants.ATTR_CERTIFICATE_STATUS; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; +import static org.apache.atlas.repository.Constants.ATTR_CONTRACT; import static org.apache.atlas.type.AtlasTypeUtil.getAtlasObjectId; public class ContractPreProcessor extends AbstractContractPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(ContractPreProcessor.class); - public static final String ATTR_CONTRACT = "dataContractJson"; public static final String ATTR_VERSION = "dataContractVersion"; public static final String REL_ATTR_GOVERNED_ASSET = "dataContractAsset"; public static final String REL_ATTR_GOVERNED_ASSET_CERTIFIED = "dataContractAssetCertified"; @@ -102,7 +102,9 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con String contractString = (String) entity.getAttribute(ATTR_CONTRACT); DataContract contract = DataContract.deserialize(contractString); - AtlasEntityWithExtInfo associatedAsset = getAssociatedAsset(contractQName, contract.type.name()); + String datasetQName = contractQName.substring(0, contractQName.lastIndexOf('/')); + contractQName = String.format("%s/%s/%s", datasetQName, contract.type.name(), CONTRACT_QUALIFIED_NAME_SUFFIX); + AtlasEntityWithExtInfo associatedAsset = getAssociatedAsset(datasetQName, contract.type.name()); authorizeContractCreateOrUpdate(entity, associatedAsset); @@ -110,7 +112,7 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con contractString = DataContract.serialize(contract); entity.setAttribute(ATTR_CONTRACT, contractString); - ContractVersionUtils versionUtil = new ContractVersionUtils(entity, context, entityRetriever, typeRegistry, entityStore, graph, discovery); + ContractVersionUtils versionUtil = new ContractVersionUtils(contractQName, context, entityRetriever, typeRegistry, entityStore, graph, discovery); AtlasEntity currentVersionEntity = versionUtil.getCurrentVersion(); int newVersionNumber = 1; if (currentVersionEntity != null) { @@ -138,7 +140,7 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con } } - entity.setAttribute(QUALIFIED_NAME, String.format("%s/%s/V%s", contractQName, VERSION_PREFIX, newVersionNumber)); + entity.setAttribute(QUALIFIED_NAME, String.format("%s/V%s", contractQName, newVersionNumber)); entity.setAttribute(ATTR_VERSION, newVersionNumber); entity.setRelationshipAttribute(REL_ATTR_GOVERNED_ASSET, getAtlasObjectId(associatedAsset.getEntity())); if (Objects.equals(entity.getAttribute(ATTR_CERTIFICATE_STATUS), DataContract.STATUS.VERIFIED.name()) ) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java index e803e51b1f..149ec2b6ea 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java @@ -23,14 +23,13 @@ public class ContractVersionUtils { public final EntityGraphRetriever entityRetriever; private final AtlasTypeRegistry atlasTypeRegistry; private AtlasEntityStore entityStore; - - private AtlasEntity entity; + private String contractQName; public final AtlasGraph graph; private List versionList; private EntityDiscoveryService discovery; - public ContractVersionUtils(AtlasEntity entity, EntityMutationContext context, EntityGraphRetriever entityRetriever, + public ContractVersionUtils(String contractQName, EntityMutationContext context, EntityGraphRetriever entityRetriever, AtlasTypeRegistry atlasTypeRegistry, AtlasEntityStore entityStore, AtlasGraph graph, EntityDiscoveryService discovery) { this.context = context; @@ -38,13 +37,11 @@ public ContractVersionUtils(AtlasEntity entity, EntityMutationContext context, E this.atlasTypeRegistry = atlasTypeRegistry; this.graph = graph; this.entityStore = entityStore; - this.entity = entity; + this.contractQName = contractQName; this.discovery = discovery; } private void extractAllVersions() throws AtlasBaseException { - String contractQName = (String) entity.getAttribute(QUALIFIED_NAME); - String datasetQName = contractQName.substring(0, contractQName.lastIndexOf("/contract")); List ret = new ArrayList<>(); IndexSearchParams indexSearchParams = new IndexSearchParams(); @@ -52,11 +49,14 @@ private void extractAllVersions() throws AtlasBaseException { List mustClauseList = new ArrayList(); mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", CONTRACT_ENTITY_TYPE))); - mustClauseList.add(mapOf("wildcard", mapOf(QUALIFIED_NAME, String.format("%s/contract/version/*", datasetQName)))); + mustClauseList.add(mapOf("wildcard", mapOf(QUALIFIED_NAME, String.format("%s/*", contractQName)))); dsl.put("query", mapOf("bool", mapOf("must", mustClauseList))); + Set attributes = new HashSet<>(); + attributes.add(ATTR_CONTRACT); indexSearchParams.setDsl(dsl); + indexSearchParams.setAttributes(attributes); indexSearchParams.setSuppressLogs(true); AtlasSearchResult result = discovery.directIndexSearch(indexSearchParams); From 8185bfe0ad29e6b6831159220f56cb1d7a50a7a1 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Wed, 24 Apr 2024 12:33:11 +0530 Subject: [PATCH 13/25] Add asset guid in DataContract asset and minor fixes --- .../v2/preprocessor/contract/ContractPreProcessor.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index b1c2474d41..e86c38514b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -33,7 +33,8 @@ public class ContractPreProcessor extends AbstractContractPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(ContractPreProcessor.class); public static final String ATTR_VERSION = "dataContractVersion"; - public static final String REL_ATTR_GOVERNED_ASSET = "dataContractAsset"; + public static final String ATTR_ASSET_GUID = "dataContractAssetGuid"; + public static final String REL_ATTR_GOVERNED_ASSET = "dataContractAssetLatest"; public static final String REL_ATTR_GOVERNED_ASSET_CERTIFIED = "dataContractAssetCertified"; public static final String REL_ATTR_PREVIOUS_VERSION = "dataContractPreviousVersion"; public static final String ASSET_ATTR_HAS_CONTRACT = "hasContract"; @@ -127,7 +128,7 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con } else if (isEqualContract(contractString, (String) currentVersionEntity.getAttribute(ATTR_CONTRACT))) { // No change in contract, metadata changed updateExistingVersion(context, entity, currentVersionEntity); - return; + newVersionNumber = currentVersionNumber; } else { // contract changed (metadata might/not changed). Create new version. newVersionNumber = currentVersionNumber + 1; @@ -142,6 +143,7 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con } entity.setAttribute(QUALIFIED_NAME, String.format("%s/V%s", contractQName, newVersionNumber)); entity.setAttribute(ATTR_VERSION, newVersionNumber); + entity.setAttribute(ATTR_ASSET_GUID, associatedAsset.getEntity().getGuid()); entity.setRelationshipAttribute(REL_ATTR_GOVERNED_ASSET, getAtlasObjectId(associatedAsset.getEntity())); if (Objects.equals(entity.getAttribute(ATTR_CERTIFICATE_STATUS), DataContract.STATUS.VERIFIED.name()) ) { entity.setRelationshipAttribute(REL_ATTR_GOVERNED_ASSET_CERTIFIED, getAtlasObjectId(associatedAsset.getEntity())); From e863c1369545ad2a0bda17bc3e3b25e93100dc06 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Wed, 24 Apr 2024 14:13:52 +0530 Subject: [PATCH 14/25] Inconsistency in relationship fixed for data contract<>asset --- .../contract/ContractPreProcessor.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index e86c38514b..17ee122301 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -34,8 +34,8 @@ public class ContractPreProcessor extends AbstractContractPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(ContractPreProcessor.class); public static final String ATTR_VERSION = "dataContractVersion"; public static final String ATTR_ASSET_GUID = "dataContractAssetGuid"; - public static final String REL_ATTR_GOVERNED_ASSET = "dataContractAssetLatest"; - public static final String REL_ATTR_GOVERNED_ASSET_CERTIFIED = "dataContractAssetCertified"; + public static final String REL_ATTR_LATEST_CONTRACT = "dataContractLatest"; + public static final String REL_ATTR_GOVERNED_ASSET_CERTIFIED = "dataContractLatestCertified"; public static final String REL_ATTR_PREVIOUS_VERSION = "dataContractPreviousVersion"; public static final String ASSET_ATTR_HAS_CONTRACT = "hasContract"; public static final String ASSET_ATTR_DESCRIPTION = "description"; @@ -144,10 +144,6 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con entity.setAttribute(QUALIFIED_NAME, String.format("%s/V%s", contractQName, newVersionNumber)); entity.setAttribute(ATTR_VERSION, newVersionNumber); entity.setAttribute(ATTR_ASSET_GUID, associatedAsset.getEntity().getGuid()); - entity.setRelationshipAttribute(REL_ATTR_GOVERNED_ASSET, getAtlasObjectId(associatedAsset.getEntity())); - if (Objects.equals(entity.getAttribute(ATTR_CERTIFICATE_STATUS), DataContract.STATUS.VERIFIED.name()) ) { - entity.setRelationshipAttribute(REL_ATTR_GOVERNED_ASSET_CERTIFIED, getAtlasObjectId(associatedAsset.getEntity())); - } datasetAttributeSync(context, associatedAsset.getEntity(), contract, entity); @@ -244,6 +240,13 @@ private void datasetAttributeSync(EntityMutationContext context, AtlasEntity ass if (associatedAsset.getAttribute(ASSET_ATTR_HAS_CONTRACT) == null || associatedAsset.getAttribute(ASSET_ATTR_HAS_CONTRACT).equals(false)) { entity.setAttribute(ASSET_ATTR_HAS_CONTRACT, true); } + + // Update relationship with contract + entity.setRelationshipAttribute(REL_ATTR_LATEST_CONTRACT, getAtlasObjectId(contractAsset)); + if (Objects.equals(contractAsset.getAttribute(ATTR_CERTIFICATE_STATUS), DataContract.STATUS.VERIFIED.name()) ) { + entity.setRelationshipAttribute(REL_ATTR_GOVERNED_ASSET_CERTIFIED, getAtlasObjectId(contractAsset)); + } + AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(entity.getGuid()); AtlasEntityType entityType = ensureEntityType(entity.getTypeName()); AtlasEntityComparator entityComparator = new AtlasEntityComparator(typeRegistry, entityRetriever, context.getGuidAssignments(), true, true); From 297152040620b62e366dda96f0e5d48b618edafd Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Wed, 24 Apr 2024 17:04:09 +0530 Subject: [PATCH 15/25] Removed maven release for data-contract branch --- .github/workflows/maven.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 39703359df..0977cb36a2 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,7 +26,6 @@ on: - development - master - lineageondemand - - data-contract jobs: build: From 4cd524d6db907106aaad0cee4c2f924a9b52cf54 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Fri, 26 Apr 2024 16:14:15 +0530 Subject: [PATCH 16/25] Change datasource to data_source in DataContract --- .../graph/v2/preprocessor/contract/ContractVersionUtils.java | 1 + .../store/graph/v2/preprocessor/contract/DataContract.java | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java index 149ec2b6ea..3c25937aef 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java @@ -54,6 +54,7 @@ private void extractAllVersions() throws AtlasBaseException { dsl.put("query", mapOf("bool", mapOf("must", mustClauseList))); Set attributes = new HashSet<>(); attributes.add(ATTR_CONTRACT); + attributes.add(ATTR_CERTIFICATE_STATUS); indexSearchParams.setDsl(dsl); indexSearchParams.setAttributes(attributes); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java index d96c885c4f..ebb4144530 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java @@ -21,7 +21,7 @@ @JsonIgnoreProperties(ignoreUnknown = true) -@JsonPropertyOrder({"kind", "status", "template_version", "datasource", "dataset", "type", "description", "owners", +@JsonPropertyOrder({"kind", "status", "template_version", "data_source", "dataset", "type", "description", "owners", "tags", "certificate", "columns"}) public class DataContract { @Valid @NotNull @@ -32,7 +32,7 @@ public class DataContract { @JsonProperty(value = "template_version", defaultValue = "0.0.1") public String templateVersion; @Valid @NotNull - public String datasource; + public String data_source; @Valid @NotNull public String dataset; @Valid @NotNull From 8ed00b3f9b32f726056debd54d021a2070cfbe30 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Sat, 27 Apr 2024 15:16:25 +0530 Subject: [PATCH 17/25] Merge create and update AuthPolicy for DataContract --- .../policies/bootstrap_entity_policies.json | 40 ++----------------- 1 file changed, 4 insertions(+), 36 deletions(-) diff --git a/addons/policies/bootstrap_entity_policies.json b/addons/policies/bootstrap_entity_policies.json index ac8629faeb..70d9b07565 100644 --- a/addons/policies/bootstrap_entity_policies.json +++ b/addons/policies/bootstrap_entity_policies.json @@ -3005,40 +3005,6 @@ ] } }, - { - "typeName": "AuthPolicy", - "attributes": - { - "name": "CREATE_DATA_CONTRACT", - "qualifiedName": "CREATE_DATA_CONTRACT", - "policyCategory": "bootstrap", - "policySubCategory": "default", - "policyServiceName": "atlas", - "policyType": "allow", - "policyPriority": 1, - "policyUsers": - [], - "policyGroups": - [], - "policyRoles": - [ - "$admin", - "$member", - "$api-token-default-access" - ], - "policyResourceCategory": "ENTITY", - "policyResources": - [ - "entity-type:DataContract", - "entity-classification:*", - "entity:*" - ], - "policyActions": - [ - "entity-create" - ] - } - }, { "typeName": "AuthPolicy", "attributes": @@ -3077,8 +3043,9 @@ "typeName": "AuthPolicy", "attributes": { - "name": "UPDATE_DATA_CONTRACT", - "qualifiedName": "UPDATE_DATA_CONTRACT", + "name": "CU_DATA_CONTRACT", + "qualifiedName": "CU_DATA_CONTRACT", + "description": "cu allow for data contract", "policyCategory": "bootstrap", "policySubCategory": "default", "policyServiceName": "atlas", @@ -3103,6 +3070,7 @@ ], "policyActions": [ + "entity-create", "entity-update" ] } From 5434342a4af7fc6a7ae283636d85adced53e0041 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Sat, 27 Apr 2024 17:40:47 +0530 Subject: [PATCH 18/25] Update DataContract attributes from public to private and create getter for needed attributes --- .../contract/ContractPreProcessor.java | 20 +++--- .../preprocessor/contract/DataContract.java | 62 ++++++++++--------- 2 files changed, 42 insertions(+), 40 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index 17ee122301..b6d333536d 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -77,12 +77,10 @@ private void processUpdateContract(AtlasEntity entity, EntityMutationContext con String contractString = (String) entity.getAttribute(ATTR_CONTRACT); AtlasVertex vertex = context.getVertex(entity.getGuid()); AtlasEntity existingContractEntity = entityRetriever.toAtlasEntity(vertex); - // TODO: Check for qualifiedName to understand if a particular version is getting updated or duplicate contract in payload if (!isEqualContract(contractString, (String) existingContractEntity.getAttribute(ATTR_CONTRACT))) { // Update the same asset(entity) throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Can't update a specific version of contract"); } - // Add cases for update in status field and certificateStatus } private void processCreateContract(AtlasEntity entity, EntityMutationContext context) throws AtlasBaseException { /* @@ -104,8 +102,8 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con String contractString = (String) entity.getAttribute(ATTR_CONTRACT); DataContract contract = DataContract.deserialize(contractString); String datasetQName = contractQName.substring(0, contractQName.lastIndexOf('/')); - contractQName = String.format("%s/%s/%s", datasetQName, contract.type.name(), CONTRACT_QUALIFIED_NAME_SUFFIX); - AtlasEntityWithExtInfo associatedAsset = getAssociatedAsset(datasetQName, contract.type.name()); + contractQName = String.format("%s/%s/%s", datasetQName, contract.getType().name(), CONTRACT_QUALIFIED_NAME_SUFFIX); + AtlasEntityWithExtInfo associatedAsset = getAssociatedAsset(datasetQName, contract.getType().name()); authorizeContractCreateOrUpdate(entity, associatedAsset); @@ -219,13 +217,13 @@ private void contractAttributeSync(AtlasEntity entity, DataContract contract) th VERIFIED - stat -> VERIFIED > */ - if (Objects.equals(entity.getAttribute(ATTR_CERTIFICATE_STATUS), DataContract.STATUS.VERIFIED.name())) { - contract.setStatus(String.valueOf(DataContract.STATUS.VERIFIED)); - } else if (Objects.equals(contract.getStatus(), DataContract.STATUS.VERIFIED)) { - entity.setAttribute(ATTR_CERTIFICATE_STATUS, DataContract.STATUS.VERIFIED.name()); + if (Objects.equals(entity.getAttribute(ATTR_CERTIFICATE_STATUS), DataContract.Status.VERIFIED.name())) { + contract.setStatus(String.valueOf(DataContract.Status.VERIFIED)); + } else if (Objects.equals(contract.getStatus(), DataContract.Status.VERIFIED)) { + entity.setAttribute(ATTR_CERTIFICATE_STATUS, DataContract.Status.VERIFIED.name()); } else { - entity.setAttribute(ATTR_CERTIFICATE_STATUS, DataContract.STATUS.DRAFT); - contract.setStatus(String.valueOf(DataContract.STATUS.DRAFT)); + entity.setAttribute(ATTR_CERTIFICATE_STATUS, DataContract.Status.DRAFT); + contract.setStatus(String.valueOf(DataContract.Status.DRAFT)); } } @@ -243,7 +241,7 @@ private void datasetAttributeSync(EntityMutationContext context, AtlasEntity ass // Update relationship with contract entity.setRelationshipAttribute(REL_ATTR_LATEST_CONTRACT, getAtlasObjectId(contractAsset)); - if (Objects.equals(contractAsset.getAttribute(ATTR_CERTIFICATE_STATUS), DataContract.STATUS.VERIFIED.name()) ) { + if (Objects.equals(contractAsset.getAttribute(ATTR_CERTIFICATE_STATUS), DataContract.Status.VERIFIED.name()) ) { entity.setRelationshipAttribute(REL_ATTR_GOVERNED_ASSET_CERTIFIED, getAtlasObjectId(contractAsset)); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java index ebb4144530..f8af5d4a61 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java @@ -24,42 +24,49 @@ @JsonPropertyOrder({"kind", "status", "template_version", "data_source", "dataset", "type", "description", "owners", "tags", "certificate", "columns"}) public class DataContract { - @Valid @NotNull - public String kind; + private static final String KIND_VALUE = "DataContract"; + private static final Pattern versionPattern = Pattern.compile("^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$"); + private static final ObjectMapper objectMapper = new ObjectMapper(); - public STATUS status; + @Valid @NotNull + private String kind; + private Status status; @JsonProperty(value = "template_version", defaultValue = "0.0.1") - public String templateVersion; + private String templateVersion; @Valid @NotNull - public String data_source; + private String data_source; @Valid @NotNull - public String dataset; + private String dataset; @Valid @NotNull - public DATASET_TYPE type; - public String description; - public List owners; - public List tags; - public String certificate; + private DatasetType type; + private String description; + private List owners; + private List tags; + private String certificate; @Valid - public List columns; + private List columns; private Map unknownFields = new HashMap<>(); @JsonSetter("type") public void setType(String type) throws AtlasBaseException { try { - this.type = DATASET_TYPE.from(type); + this.type = DatasetType.from(type); } catch (IllegalArgumentException | AtlasBaseException ex) { - throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "type " + type + " is inappropriate. Accepted values: " + Arrays.toString(DATASET_TYPE.values())); + throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "type " + type + " is inappropriate. Accepted values: " + Arrays.toString(DatasetType.values())); } } - public enum DATASET_TYPE { + public DatasetType getType() { + return type; + } + + public enum DatasetType { @JsonProperty("Table") Table, @JsonProperty("View") View, @JsonProperty("MaterialisedView") MaterialisedView; - public static DATASET_TYPE from(String s) throws AtlasBaseException { + public static DatasetType from(String s) throws AtlasBaseException { switch (s.toLowerCase()) { case "table": @@ -73,16 +80,16 @@ public static DATASET_TYPE from(String s) throws AtlasBaseException { } } } - public STATUS getStatus() { + public Status getStatus() { return status; } @JsonSetter("status") public void setStatus(String status) throws AtlasBaseException { try { - this.status = STATUS.from(status); + this.status = Status.from(status); } catch (IllegalArgumentException ex) { - throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "status " + status + " is inappropriate. Accepted values: " + Arrays.toString(STATUS.values())); + throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "status " + status + " is inappropriate. Accepted values: " + Arrays.toString(Status.values())); } } @@ -96,11 +103,11 @@ public Map getUnknownFields() { return unknownFields; } - public enum STATUS { + public enum Status { @JsonProperty("DRAFT") DRAFT, @JsonProperty("VERIFIED") VERIFIED; - public static STATUS from(String s) { + public static Status from(String s) { if(StringUtils.isEmpty(s)) { return DRAFT; } @@ -119,7 +126,7 @@ public static STATUS from(String s) { @JsonSetter("kind") public void setKind(String kind) throws AtlasBaseException { - if (!"DataContract".equals(kind)) { + if (!KIND_VALUE.equals(kind)) { throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "kind " + kind + " is inappropriate."); } this.kind = kind; @@ -133,7 +140,6 @@ public void setTemplateVersion(String templateVersion) throws AtlasBaseException } private boolean isSemVer(String version) { - Pattern versionPattern = Pattern.compile("^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$"); Matcher matcher = versionPattern.matcher(version); return matcher.matches(); } @@ -159,13 +165,13 @@ public Map getUnknownFields() { @JsonPropertyOrder({"name", "description", "data_type"}) public static final class Column { @NotNull - public String name; + private String name; - public String description; + private String description; - public boolean is_primary; + private boolean is_primary; - public String data_type; + private String data_type; private Map unknownFields = new HashMap<>(); @JsonAnySetter @@ -185,7 +191,6 @@ public static DataContract deserialize(String contractString) throws AtlasBaseEx throw new AtlasBaseException(BAD_REQUEST, "Missing attribute: contract."); } - ObjectMapper objectMapper = new ObjectMapper(); objectMapper.enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS); DataContract contract; try { @@ -215,7 +220,6 @@ public void validate() throws AtlasBaseException { public static String serialize(DataContract contract) throws AtlasBaseException { try { - ObjectMapper objectMapper = new ObjectMapper(); objectMapper.enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS); return objectMapper.writeValueAsString(contract); } catch (JsonProcessingException ex) { From 09c500a0c1b59e7cbbaacb26aea704e8f4d05e82 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Sat, 27 Apr 2024 20:13:59 +0530 Subject: [PATCH 19/25] Add setter for private attributes and formatted the file --- .../preprocessor/contract/DataContract.java | 153 +++++++++++------- 1 file changed, 98 insertions(+), 55 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java index f8af5d4a61..62887c3b23 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java @@ -31,36 +31,42 @@ public class DataContract { @Valid @NotNull private String kind; private Status status; - @JsonProperty(value = "template_version", defaultValue = "0.0.1") private String templateVersion; @Valid @NotNull - private String data_source; + private String data_source; @Valid @NotNull - private String dataset; + private String dataset; @Valid @NotNull - private DatasetType type; - private String description; - private List owners; - private List tags; - private String certificate; + private DatasetType type; + private String description; + private List owners; + private List tags; + private String certificate; @Valid - private List columns; - private Map unknownFields = new HashMap<>(); + private List columns; + private final Map unknownFields = new HashMap<>(); - @JsonSetter("type") - public void setType(String type) throws AtlasBaseException { - try { - this.type = DatasetType.from(type); - } catch (IllegalArgumentException | AtlasBaseException ex) { - throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "type " + type + " is inappropriate. Accepted values: " + Arrays.toString(DatasetType.values())); - } - } + public enum Status { + @JsonProperty("DRAFT") DRAFT, + @JsonProperty("VERIFIED") VERIFIED; - public DatasetType getType() { - return type; - } + public static Status from(String s) { + if(StringUtils.isEmpty(s)) { + return DRAFT; + } + switch (s.toLowerCase()) { + case "draft": + return DRAFT; + case "verified": + return VERIFIED; + + default: + return DRAFT; + } + } + } public enum DatasetType { @JsonProperty("Table") Table, @JsonProperty("View") View, @@ -80,22 +86,13 @@ public static DatasetType from(String s) throws AtlasBaseException { } } } + public Status getStatus() { return status; } - @JsonSetter("status") - public void setStatus(String status) throws AtlasBaseException { - try { - this.status = Status.from(status); - } catch (IllegalArgumentException ex) { - throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "status " + status + " is inappropriate. Accepted values: " + Arrays.toString(Status.values())); - } - } - - @JsonAnySetter - public void setUnknownFields(String key, Object value) { - unknownFields.put(key, value); + public DatasetType getType() { + return type; } @JsonAnyGetter @@ -103,27 +100,6 @@ public Map getUnknownFields() { return unknownFields; } - public enum Status { - @JsonProperty("DRAFT") DRAFT, - @JsonProperty("VERIFIED") VERIFIED; - - public static Status from(String s) { - if(StringUtils.isEmpty(s)) { - return DRAFT; - } - switch (s.toLowerCase()) { - case "draft": - return DRAFT; - - case "verified": - return VERIFIED; - - default: - return DRAFT; - } - } - } - @JsonSetter("kind") public void setKind(String kind) throws AtlasBaseException { if (!KIND_VALUE.equals(kind)) { @@ -132,6 +108,15 @@ public void setKind(String kind) throws AtlasBaseException { this.kind = kind; } + @JsonSetter("status") + public void setStatus(String status) throws AtlasBaseException { + try { + this.status = Status.from(status); + } catch (IllegalArgumentException ex) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "status " + status + " is inappropriate. Accepted values: " + Arrays.toString(Status.values())); + } + } + public void setTemplateVersion(String templateVersion) throws AtlasBaseException { if (!isSemVer(templateVersion)) { throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "Invalid template_version syntax"); @@ -139,6 +124,49 @@ public void setTemplateVersion(String templateVersion) throws AtlasBaseException this.templateVersion = templateVersion; } + public void setDataSource(String data_source) { + this.data_source = data_source; + } + + public void setDataset(String dataset) { + this.dataset = dataset; + } + + @JsonSetter("type") + public void setType(String type) throws AtlasBaseException { + try { + this.type = DatasetType.from(type); + } catch (IllegalArgumentException | AtlasBaseException ex) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "type " + type + " is inappropriate. Accepted values: " + Arrays.toString(DatasetType.values())); + } + } + + public void setDescription(String description) { + this.description = description; + } + + public void setOwners(List owners) { + this.owners = owners; + } + + public void setTags(List tags) { + this.tags = tags; + } + + public void setCertificate(String certificate) { + this.certificate = certificate; + } + + public void setColumns(List columns) { + this.columns = columns; + } + + @JsonAnySetter + public void setUnknownFields(String key, Object value) { + unknownFields.put(key, value); + } + + private boolean isSemVer(String version) { Matcher matcher = versionPattern.matcher(version); return matcher.matches(); @@ -161,6 +189,7 @@ public Map getUnknownFields() { } } + @JsonIgnoreProperties(ignoreUnknown = true) @JsonPropertyOrder({"name", "description", "data_type"}) public static final class Column { @@ -183,6 +212,21 @@ public Map getUnknownFields() { return unknownFields; } + public void setName(String name) { + this.name = name; + } + + public void setDescription(String description) { + this.description = description; + } + + public void setIs_primary(boolean is_primary) { + this.is_primary = is_primary; + } + + public void setData_type(String data_type) { + this.data_type = data_type; + } } public static DataContract deserialize(String contractString) throws AtlasBaseException { @@ -227,6 +271,5 @@ public static String serialize(DataContract contract) throws AtlasBaseException } } - } From d78cdc14ad043336599c636b41c5454cd8c966dd Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Sun, 28 Apr 2024 09:45:57 +0530 Subject: [PATCH 20/25] Address PR review changes * Remove version util and moved getCurrentVersion method to preprocessor * Remove unused varibles * Optimize fetching latest version of contract * --- .../apache/atlas/repository/Constants.java | 2 + .../AbstractContractPreProcessor.java | 2 - .../contract/ContractPreProcessor.java | 55 +++++++++--- .../contract/ContractVersionUtils.java | 85 ------------------- 4 files changed, 46 insertions(+), 98 deletions(-) delete mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index 2152921981..8df6e0bbaa 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -167,6 +167,8 @@ public final class Constants { * Contract */ public static final String CONTRACT_ENTITY_TYPE = "DataContract"; + public static final String ATTR_CONTRACT_VERSION = "dataContractVersion"; + /** * Lineage relations. diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java index 9b063d38f5..e167a00b9a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java @@ -79,8 +79,6 @@ AtlasEntity.AtlasEntityWithExtInfo getAssociatedAsset(String datasetQName, Strin AtlasVertex entityVertex = AtlasGraphUtilsV2.getVertexByUniqueAttributes(graph, entityType, uniqAttributes); - EntityGraphRetriever entityRetriever = new EntityGraphRetriever(graph, typeRegistry, true); - AtlasEntity.AtlasEntityWithExtInfo ret = entityRetriever.toAtlasEntityWithExtInfo(entityVertex); if (ret == null) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index b6d333536d..a60ed4ef09 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -7,6 +7,8 @@ import org.apache.atlas.RequestContext; import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.discovery.AtlasSearchResult; +import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; import org.apache.atlas.model.instance.AtlasStruct; @@ -25,14 +27,12 @@ import java.util.*; import static org.apache.atlas.AtlasErrorCode.*; -import static org.apache.atlas.repository.Constants.ATTR_CERTIFICATE_STATUS; -import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; -import static org.apache.atlas.repository.Constants.ATTR_CONTRACT; +import static org.apache.atlas.repository.Constants.*; +import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; import static org.apache.atlas.type.AtlasTypeUtil.getAtlasObjectId; public class ContractPreProcessor extends AbstractContractPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(ContractPreProcessor.class); - public static final String ATTR_VERSION = "dataContractVersion"; public static final String ATTR_ASSET_GUID = "dataContractAssetGuid"; public static final String REL_ATTR_LATEST_CONTRACT = "dataContractLatest"; public static final String REL_ATTR_GOVERNED_ASSET_CERTIFIED = "dataContractLatestCertified"; @@ -45,7 +45,7 @@ public class ContractPreProcessor extends AbstractContractPreProcessor { public static final String CONTRACT_ATTR_STATUS = "status"; private final AtlasEntityStore entityStore; private final boolean storeDifferentialAudits; - private EntityDiscoveryService discovery; + private final EntityDiscoveryService discovery; @@ -107,12 +107,13 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con authorizeContractCreateOrUpdate(entity, associatedAsset); - contractAttributeSync(entity, contract); - contractString = DataContract.serialize(contract); + boolean contractSync = syncContractCertificateStatus(entity, contract); + if (contractSync) { + contractString = DataContract.serialize(contract); + } entity.setAttribute(ATTR_CONTRACT, contractString); - ContractVersionUtils versionUtil = new ContractVersionUtils(contractQName, context, entityRetriever, typeRegistry, entityStore, graph, discovery); - AtlasEntity currentVersionEntity = versionUtil.getCurrentVersion(); + AtlasEntity currentVersionEntity = getCurrentVersion(contractQName); int newVersionNumber = 1; if (currentVersionEntity != null) { // Contract already exist @@ -140,7 +141,7 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con } } entity.setAttribute(QUALIFIED_NAME, String.format("%s/V%s", contractQName, newVersionNumber)); - entity.setAttribute(ATTR_VERSION, newVersionNumber); + entity.setAttribute(ATTR_CONTRACT_VERSION, newVersionNumber); entity.setAttribute(ATTR_ASSET_GUID, associatedAsset.getEntity().getGuid()); datasetAttributeSync(context, associatedAsset.getEntity(), contract, entity); @@ -191,6 +192,34 @@ private void updateExistingVersion(EntityMutationContext context, AtlasEntity en } + public AtlasEntity getCurrentVersion(String contractQName) throws AtlasBaseException { + IndexSearchParams indexSearchParams = new IndexSearchParams(); + Map dsl = new HashMap<>(); + int size = 1; + + List> mustClauseList = new ArrayList<>(); + mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", CONTRACT_ENTITY_TYPE))); + mustClauseList.add(mapOf("wildcard", mapOf(QUALIFIED_NAME, String.format("%s/*", contractQName)))); + + dsl.put("query", mapOf("bool", mapOf("must", mustClauseList))); + dsl.put("sort", Collections.singletonList(mapOf(ATTR_CONTRACT_VERSION, mapOf("order", "desc")))); + dsl.put("size", size); + + final Set attributes = new HashSet<>(); + attributes.add(ATTR_CONTRACT); + attributes.add(ATTR_CERTIFICATE_STATUS); + + indexSearchParams.setDsl(dsl); + indexSearchParams.setAttributes(attributes); + indexSearchParams.setSuppressLogs(true); + + AtlasSearchResult result = discovery.directIndexSearch(indexSearchParams); + if (result == null) { + return null; + } + return new AtlasEntity(result.getEntities().get(0)); + } + private void removeCreatingVertex(EntityMutationContext context, AtlasEntity entity) throws AtlasBaseException { context.getCreatedEntities().remove(entity); try { @@ -204,7 +233,8 @@ private void removeCreatingVertex(EntityMutationContext context, AtlasEntity ent } - private void contractAttributeSync(AtlasEntity entity, DataContract contract) throws AtlasBaseException { + private boolean syncContractCertificateStatus(AtlasEntity entity, DataContract contract) throws AtlasBaseException { + boolean contractSync = false; // Sync certificateStatus if (!Objects.equals(entity.getAttribute(ATTR_CERTIFICATE_STATUS), contract.getStatus().name())) { /* @@ -219,14 +249,17 @@ private void contractAttributeSync(AtlasEntity entity, DataContract contract) th */ if (Objects.equals(entity.getAttribute(ATTR_CERTIFICATE_STATUS), DataContract.Status.VERIFIED.name())) { contract.setStatus(String.valueOf(DataContract.Status.VERIFIED)); + contractSync = true; } else if (Objects.equals(contract.getStatus(), DataContract.Status.VERIFIED)) { entity.setAttribute(ATTR_CERTIFICATE_STATUS, DataContract.Status.VERIFIED.name()); } else { entity.setAttribute(ATTR_CERTIFICATE_STATUS, DataContract.Status.DRAFT); contract.setStatus(String.valueOf(DataContract.Status.DRAFT)); + contractSync = true; } } + return contractSync; } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java deleted file mode 100644 index 3c25937aef..0000000000 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractVersionUtils.java +++ /dev/null @@ -1,85 +0,0 @@ -package org.apache.atlas.repository.store.graph.v2.preprocessor.contract; - -import org.apache.atlas.discovery.EntityDiscoveryService; -import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.discovery.AtlasSearchResult; -import org.apache.atlas.model.discovery.IndexSearchParams; -import org.apache.atlas.model.instance.AtlasEntity; -import org.apache.atlas.model.instance.AtlasEntityHeader; -import org.apache.atlas.repository.graphdb.AtlasGraph; -import org.apache.atlas.repository.store.graph.AtlasEntityStore; -import org.apache.atlas.repository.store.graph.v2.*; -import org.apache.atlas.type.AtlasTypeRegistry; - -import java.util.*; - -import static org.apache.atlas.repository.Constants.*; -import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; - - -public class ContractVersionUtils { - - private final EntityMutationContext context; - public final EntityGraphRetriever entityRetriever; - private final AtlasTypeRegistry atlasTypeRegistry; - private AtlasEntityStore entityStore; - private String contractQName; - - public final AtlasGraph graph; - private List versionList; - private EntityDiscoveryService discovery; - - public ContractVersionUtils(String contractQName, EntityMutationContext context, EntityGraphRetriever entityRetriever, - AtlasTypeRegistry atlasTypeRegistry, AtlasEntityStore entityStore, AtlasGraph graph, - EntityDiscoveryService discovery) { - this.context = context; - this.entityRetriever = entityRetriever; - this.atlasTypeRegistry = atlasTypeRegistry; - this.graph = graph; - this.entityStore = entityStore; - this.contractQName = contractQName; - this.discovery = discovery; - } - - private void extractAllVersions() throws AtlasBaseException { - List ret = new ArrayList<>(); - - IndexSearchParams indexSearchParams = new IndexSearchParams(); - Map dsl = new HashMap<>(); - - List mustClauseList = new ArrayList(); - mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", CONTRACT_ENTITY_TYPE))); - mustClauseList.add(mapOf("wildcard", mapOf(QUALIFIED_NAME, String.format("%s/*", contractQName)))); - - dsl.put("query", mapOf("bool", mapOf("must", mustClauseList))); - Set attributes = new HashSet<>(); - attributes.add(ATTR_CONTRACT); - attributes.add(ATTR_CERTIFICATE_STATUS); - - indexSearchParams.setDsl(dsl); - indexSearchParams.setAttributes(attributes); - indexSearchParams.setSuppressLogs(true); - - AtlasSearchResult result = discovery.directIndexSearch(indexSearchParams); - if (result != null) { - ret = result.getEntities(); - } - this.versionList = ret; - } - - public AtlasEntity getCurrentVersion() throws AtlasBaseException { - if (this.versionList == null) { - extractAllVersions(); - } - if (this.versionList == null) { - return null; - } - Collections.sort(this.versionList, (e1, e2) -> { - String e1QName = (String) e1.getAttribute(QUALIFIED_NAME); - String e2QName = (String) e2.getAttribute(QUALIFIED_NAME); - - return e2QName.compareTo(e1QName); - }); - return new AtlasEntity(this.versionList.get(0)); - } -} From 9779ffeb081bbbb4ae6f0bbea7b443fcc633b906 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Mon, 29 Apr 2024 22:03:43 +0530 Subject: [PATCH 21/25] Address code review comments * Remove purgebyId call and remove skip auth in the purge method * Make DataContract attributes public for serialisation and deserialisation of JSON * Make minor changes for optimisation --- .../store/graph/v2/AtlasEntityStoreV2.java | 5 +- .../AbstractContractPreProcessor.java | 8 +-- .../contract/ContractPreProcessor.java | 33 +++++------ .../preprocessor/contract/DataContract.java | 57 +++++++------------ 4 files changed, 38 insertions(+), 65 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 8e11aaeff1..519068a9da 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -695,9 +695,8 @@ public EntityMutationResponse purgeByIds(Set guids) throws AtlasBaseExce if (CollectionUtils.isEmpty(guids)) { throw new AtlasBaseException(AtlasErrorCode.INVALID_PARAMETERS, "Guid(s) not specified"); } - if (!RequestContext.get().isSkipAuthorizationCheck()) { - AtlasAuthorizationUtils.verifyAccess(new AtlasAdminAccessRequest(AtlasPrivilege.ADMIN_PURGE), "purge entity: guids=", guids); - } + + AtlasAuthorizationUtils.verifyAccess(new AtlasAdminAccessRequest(AtlasPrivilege.ADMIN_PURGE), "purge entity: guids=", guids); Collection purgeCandidates = new ArrayList<>(); for (String guid : guids) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java index e167a00b9a..0a4521e34b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java @@ -79,13 +79,7 @@ AtlasEntity.AtlasEntityWithExtInfo getAssociatedAsset(String datasetQName, Strin AtlasVertex entityVertex = AtlasGraphUtilsV2.getVertexByUniqueAttributes(graph, entityType, uniqAttributes); - AtlasEntity.AtlasEntityWithExtInfo ret = entityRetriever.toAtlasEntityWithExtInfo(entityVertex); - - if (ret == null) { - throw new AtlasBaseException(INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND, entityType.getTypeName(), - uniqAttributes.toString()); - } - return ret; + return entityRetriever.toAtlasEntityWithExtInfo(entityVertex); } AtlasEntityType ensureEntityType(String typeName) throws AtlasBaseException { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index a60ed4ef09..f45e6ecf57 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -19,6 +19,7 @@ import org.apache.atlas.repository.store.graph.v2.*; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.commons.collections.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -47,6 +48,7 @@ public class ContractPreProcessor extends AbstractContractPreProcessor { private final boolean storeDifferentialAudits; private final EntityDiscoveryService discovery; + private final AtlasEntityComparator entityComparator; public ContractPreProcessor(AtlasGraph graph, AtlasTypeRegistry typeRegistry, @@ -57,6 +59,8 @@ public ContractPreProcessor(AtlasGraph graph, AtlasTypeRegistry typeRegistry, this.storeDifferentialAudits = storeDifferentialAudits; this.entityStore = entityStore; this.discovery = discovery; + this.entityComparator = new AtlasEntityComparator(typeRegistry, entityRetriever, null, true, true); + } @Override @@ -110,15 +114,15 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con boolean contractSync = syncContractCertificateStatus(entity, contract); if (contractSync) { contractString = DataContract.serialize(contract); + entity.setAttribute(ATTR_CONTRACT, contractString); } - entity.setAttribute(ATTR_CONTRACT, contractString); - AtlasEntity currentVersionEntity = getCurrentVersion(contractQName); - int newVersionNumber = 1; + AtlasEntity currentVersionEntity = getCurrentVersion(associatedAsset.getEntity().getGuid()); + Long newVersionNumber = 1L; if (currentVersionEntity != null) { // Contract already exist String qName = (String) currentVersionEntity.getAttribute(QUALIFIED_NAME); - int currentVersionNumber = Integer.parseInt(qName.substring(qName.lastIndexOf("/V") + 2)); + Long currentVersionNumber = (Long) currentVersionEntity.getAttribute(ATTR_CONTRACT_VERSION); List attributes = getDiffAttributes(context, entity, currentVersionEntity); if (attributes.isEmpty()) { // No changes in the contract, Not creating new version @@ -149,7 +153,6 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con } private List getDiffAttributes(EntityMutationContext context, AtlasEntity entity, AtlasEntity latestExistingVersion) throws AtlasBaseException { - AtlasEntityComparator entityComparator = new AtlasEntityComparator(typeRegistry, entityRetriever, context.getGuidAssignments(), true, true); AtlasEntityComparator.AtlasEntityDiffResult diffResult = entityComparator.getDiffResult(entity, latestExistingVersion, false); List attributesSet = new ArrayList<>(); @@ -183,23 +186,21 @@ private void updateExistingVersion(EntityMutationContext context, AtlasEntity en removeCreatingVertex(context, entity); entity.setAttribute(QUALIFIED_NAME, currentVersionEntity.getAttribute(QUALIFIED_NAME)); entity.setGuid(currentVersionEntity.getGuid()); - - AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(entity.getGuid()); - + AtlasVertex vertex = context.getVertex(entity.getGuid()); AtlasEntityType entityType = ensureEntityType(entity.getTypeName()); context.addUpdated(entity.getGuid(), entity, entityType, vertex); } - public AtlasEntity getCurrentVersion(String contractQName) throws AtlasBaseException { + public AtlasEntity getCurrentVersion(String datasetGuid) throws AtlasBaseException { IndexSearchParams indexSearchParams = new IndexSearchParams(); Map dsl = new HashMap<>(); int size = 1; List> mustClauseList = new ArrayList<>(); mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", CONTRACT_ENTITY_TYPE))); - mustClauseList.add(mapOf("wildcard", mapOf(QUALIFIED_NAME, String.format("%s/*", contractQName)))); + mustClauseList.add(mapOf("term", mapOf(ATTR_ASSET_GUID, datasetGuid))); dsl.put("query", mapOf("bool", mapOf("must", mustClauseList))); dsl.put("sort", Collections.singletonList(mapOf(ATTR_CONTRACT_VERSION, mapOf("order", "desc")))); @@ -208,13 +209,14 @@ public AtlasEntity getCurrentVersion(String contractQName) throws AtlasBaseExcep final Set attributes = new HashSet<>(); attributes.add(ATTR_CONTRACT); attributes.add(ATTR_CERTIFICATE_STATUS); + attributes.add(ATTR_CONTRACT_VERSION); indexSearchParams.setDsl(dsl); indexSearchParams.setAttributes(attributes); indexSearchParams.setSuppressLogs(true); AtlasSearchResult result = discovery.directIndexSearch(indexSearchParams); - if (result == null) { + if (result == null || CollectionUtils.isEmpty(result.getEntities())) { return null; } return new AtlasEntity(result.getEntities().get(0)); @@ -222,14 +224,7 @@ public AtlasEntity getCurrentVersion(String contractQName) throws AtlasBaseExcep private void removeCreatingVertex(EntityMutationContext context, AtlasEntity entity) throws AtlasBaseException { context.getCreatedEntities().remove(entity); - try { - RequestContext.get().setSkipAuthorizationCheck(true); - Set guids = new HashSet<>(); - guids.add(entity.getGuid()); - entityStore.purgeByIds(guids); - } finally { - RequestContext.get().setSkipAuthorizationCheck(false); - } + graph.removeVertex(context.getVertex(entity.getGuid())); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java index 62887c3b23..ae45bf2353 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java @@ -27,24 +27,27 @@ public class DataContract { private static final String KIND_VALUE = "DataContract"; private static final Pattern versionPattern = Pattern.compile("^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-((?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\\.(?:0|[1-9]\\d*|\\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\\+([0-9a-zA-Z-]+(?:\\.[0-9a-zA-Z-]+)*))?$"); private static final ObjectMapper objectMapper = new ObjectMapper(); + static { + objectMapper.enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS); + } @Valid @NotNull - private String kind; - private Status status; + public String kind; + public Status status; @JsonProperty(value = "template_version", defaultValue = "0.0.1") - private String templateVersion; + public String templateVersion; @Valid @NotNull - private String data_source; + public String data_source; @Valid @NotNull - private String dataset; + public String dataset; @Valid @NotNull - private DatasetType type; - private String description; - private List owners; - private List tags; - private String certificate; + public DatasetType type; + public String description; + public List owners; + public List tags; + public String certificate; @Valid - private List columns; + public List columns; private final Map unknownFields = new HashMap<>(); public enum Status { @@ -124,6 +127,7 @@ public void setTemplateVersion(String templateVersion) throws AtlasBaseException this.templateVersion = templateVersion; } + @JsonSetter("data_source") public void setDataSource(String data_source) { this.data_source = data_source; } @@ -132,7 +136,6 @@ public void setDataset(String dataset) { this.dataset = dataset; } - @JsonSetter("type") public void setType(String type) throws AtlasBaseException { try { this.type = DatasetType.from(type); @@ -177,7 +180,7 @@ private boolean isSemVer(String version) { public static final class BusinessTag { @NotNull public String name; - private Map unknownFields = new HashMap<>(); + private final Map unknownFields = new HashMap<>(); @JsonAnySetter public void setUnknownFields(String key, Object value) { @@ -194,14 +197,14 @@ public Map getUnknownFields() { @JsonPropertyOrder({"name", "description", "data_type"}) public static final class Column { @NotNull - private String name; + public String name; - private String description; + public String description; - private boolean is_primary; + public boolean is_primary; - private String data_type; - private Map unknownFields = new HashMap<>(); + public String data_type; + private final Map unknownFields = new HashMap<>(); @JsonAnySetter public void setUnknownFields(String key, Object value) { @@ -211,22 +214,6 @@ public void setUnknownFields(String key, Object value) { public Map getUnknownFields() { return unknownFields; } - - public void setName(String name) { - this.name = name; - } - - public void setDescription(String description) { - this.description = description; - } - - public void setIs_primary(boolean is_primary) { - this.is_primary = is_primary; - } - - public void setData_type(String data_type) { - this.data_type = data_type; - } } public static DataContract deserialize(String contractString) throws AtlasBaseException { @@ -235,7 +222,6 @@ public static DataContract deserialize(String contractString) throws AtlasBaseEx throw new AtlasBaseException(BAD_REQUEST, "Missing attribute: contract."); } - objectMapper.enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS); DataContract contract; try { contract = objectMapper.readValue(contractString, DataContract.class); @@ -264,7 +250,6 @@ public void validate() throws AtlasBaseException { public static String serialize(DataContract contract) throws AtlasBaseException { try { - objectMapper.enable(MapperFeature.ACCEPT_CASE_INSENSITIVE_ENUMS); return objectMapper.writeValueAsString(contract); } catch (JsonProcessingException ex) { throw new AtlasBaseException(JSON_ERROR, ex.getMessage()); From 191b447418c189f5bfc75e4820a2a98cc787889f Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Tue, 30 Apr 2024 08:22:51 +0530 Subject: [PATCH 22/25] Remove redundant code/arguments in contract preprocessor --- .../store/graph/v2/AtlasEntityStoreV2.java | 2 +- .../contract/ContractPreProcessor.java | 33 ++++++++----------- 2 files changed, 14 insertions(+), 21 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 519068a9da..3436ec3e10 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -1838,7 +1838,7 @@ public PreProcessor getPreProcessor(String typeName) { break; case CONTRACT_ENTITY_TYPE: - preProcessor = new ContractPreProcessor(graph, typeRegistry, entityRetriever, this, storeDifferentialAudits, discovery); + preProcessor = new ContractPreProcessor(graph, typeRegistry, entityRetriever, storeDifferentialAudits, discovery); break; } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index f45e6ecf57..47fc097042 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -15,7 +15,6 @@ import org.apache.atlas.model.instance.EntityMutations; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; -import org.apache.atlas.repository.store.graph.AtlasEntityStore; import org.apache.atlas.repository.store.graph.v2.*; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; @@ -39,12 +38,14 @@ public class ContractPreProcessor extends AbstractContractPreProcessor { public static final String REL_ATTR_GOVERNED_ASSET_CERTIFIED = "dataContractLatestCertified"; public static final String REL_ATTR_PREVIOUS_VERSION = "dataContractPreviousVersion"; public static final String ASSET_ATTR_HAS_CONTRACT = "hasContract"; - public static final String ASSET_ATTR_DESCRIPTION = "description"; - public static final String CONTRACT_QUALIFIED_NAME_SUFFIX = "contract"; - public static final String VERSION_PREFIX = "version"; public static final String CONTRACT_ATTR_STATUS = "status"; - private final AtlasEntityStore entityStore; + private static final Set contractAttributes = new HashSet<>(); + static { + contractAttributes.add(ATTR_CONTRACT); + contractAttributes.add(ATTR_CERTIFICATE_STATUS); + contractAttributes.add(ATTR_CONTRACT_VERSION); + } private final boolean storeDifferentialAudits; private final EntityDiscoveryService discovery; @@ -52,12 +53,11 @@ public class ContractPreProcessor extends AbstractContractPreProcessor { public ContractPreProcessor(AtlasGraph graph, AtlasTypeRegistry typeRegistry, - EntityGraphRetriever entityRetriever, AtlasEntityStore entityStore, + EntityGraphRetriever entityRetriever, boolean storeDifferentialAudits, EntityDiscoveryService discovery) { super(graph, typeRegistry, entityRetriever); this.storeDifferentialAudits = storeDifferentialAudits; - this.entityStore = entityStore; this.discovery = discovery; this.entityComparator = new AtlasEntityComparator(typeRegistry, entityRetriever, null, true, true); @@ -121,9 +121,8 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con Long newVersionNumber = 1L; if (currentVersionEntity != null) { // Contract already exist - String qName = (String) currentVersionEntity.getAttribute(QUALIFIED_NAME); Long currentVersionNumber = (Long) currentVersionEntity.getAttribute(ATTR_CONTRACT_VERSION); - List attributes = getDiffAttributes(context, entity, currentVersionEntity); + List attributes = getDiffAttributes(entity, currentVersionEntity); if (attributes.isEmpty()) { // No changes in the contract, Not creating new version removeCreatingVertex(context, entity); @@ -148,11 +147,11 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con entity.setAttribute(ATTR_CONTRACT_VERSION, newVersionNumber); entity.setAttribute(ATTR_ASSET_GUID, associatedAsset.getEntity().getGuid()); - datasetAttributeSync(context, associatedAsset.getEntity(), contract, entity); + datasetAttributeSync(context, associatedAsset.getEntity(), entity); } - private List getDiffAttributes(EntityMutationContext context, AtlasEntity entity, AtlasEntity latestExistingVersion) throws AtlasBaseException { + private List getDiffAttributes(AtlasEntity entity, AtlasEntity latestExistingVersion) throws AtlasBaseException { AtlasEntityComparator.AtlasEntityDiffResult diffResult = entityComparator.getDiffResult(entity, latestExistingVersion, false); List attributesSet = new ArrayList<>(); @@ -206,13 +205,8 @@ public AtlasEntity getCurrentVersion(String datasetGuid) throws AtlasBaseExcepti dsl.put("sort", Collections.singletonList(mapOf(ATTR_CONTRACT_VERSION, mapOf("order", "desc")))); dsl.put("size", size); - final Set attributes = new HashSet<>(); - attributes.add(ATTR_CONTRACT); - attributes.add(ATTR_CERTIFICATE_STATUS); - attributes.add(ATTR_CONTRACT_VERSION); - indexSearchParams.setDsl(dsl); - indexSearchParams.setAttributes(attributes); + indexSearchParams.setAttributes(contractAttributes); indexSearchParams.setSuppressLogs(true); AtlasSearchResult result = discovery.directIndexSearch(indexSearchParams); @@ -222,10 +216,9 @@ public AtlasEntity getCurrentVersion(String datasetGuid) throws AtlasBaseExcepti return new AtlasEntity(result.getEntities().get(0)); } - private void removeCreatingVertex(EntityMutationContext context, AtlasEntity entity) throws AtlasBaseException { + private void removeCreatingVertex(EntityMutationContext context, AtlasEntity entity) { context.getCreatedEntities().remove(entity); graph.removeVertex(context.getVertex(entity.getGuid())); - } private boolean syncContractCertificateStatus(AtlasEntity entity, DataContract contract) throws AtlasBaseException { @@ -258,7 +251,7 @@ private boolean syncContractCertificateStatus(AtlasEntity entity, DataContract c } - private void datasetAttributeSync(EntityMutationContext context, AtlasEntity associatedAsset, DataContract contract, AtlasEntity contractAsset) throws AtlasBaseException { + private void datasetAttributeSync(EntityMutationContext context, AtlasEntity associatedAsset, AtlasEntity contractAsset) throws AtlasBaseException { // Creating new empty AtlasEntity to update with selective attributes only AtlasEntity entity = new AtlasEntity(associatedAsset.getTypeName()); entity.setGuid(associatedAsset.getGuid()); From 9f26e464622d1c2a7ddd66097f05e32a30c106d6 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Tue, 30 Apr 2024 12:12:42 +0530 Subject: [PATCH 23/25] Add remove user added relationship to control relationship update --- .../contract/ContractPreProcessor.java | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index 47fc097042..df3efde85f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -81,6 +81,8 @@ private void processUpdateContract(AtlasEntity entity, EntityMutationContext con String contractString = (String) entity.getAttribute(ATTR_CONTRACT); AtlasVertex vertex = context.getVertex(entity.getGuid()); AtlasEntity existingContractEntity = entityRetriever.toAtlasEntity(vertex); + // No update to relationships allowed for the existing contract version + resetAllRelationshipAttributes(entity); if (!isEqualContract(contractString, (String) existingContractEntity.getAttribute(ATTR_CONTRACT))) { // Update the same asset(entity) throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Can't update a specific version of contract"); @@ -128,6 +130,7 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con removeCreatingVertex(context, entity); return; } else if (isEqualContract(contractString, (String) currentVersionEntity.getAttribute(ATTR_CONTRACT))) { + resetAllRelationshipAttributes(entity); // No change in contract, metadata changed updateExistingVersion(context, entity, currentVersionEntity); newVersionNumber = currentVersionNumber; @@ -135,6 +138,7 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con // contract changed (metadata might/not changed). Create new version. newVersionNumber = currentVersionNumber + 1; + resetAllRelationshipAttributes(entity); // Attach previous version via rel entity.setRelationshipAttribute(REL_ATTR_PREVIOUS_VERSION, getAtlasObjectId(currentVersionEntity)); AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(currentVersionEntity.getGuid()); @@ -185,7 +189,7 @@ private void updateExistingVersion(EntityMutationContext context, AtlasEntity en removeCreatingVertex(context, entity); entity.setAttribute(QUALIFIED_NAME, currentVersionEntity.getAttribute(QUALIFIED_NAME)); entity.setGuid(currentVersionEntity.getGuid()); - AtlasVertex vertex = context.getVertex(entity.getGuid()); + AtlasVertex vertex = AtlasGraphUtilsV2.findByGuid(entity.getGuid()); AtlasEntityType entityType = ensureEntityType(entity.getTypeName()); context.addUpdated(entity.getGuid(), entity, entityType, vertex); @@ -221,6 +225,18 @@ private void removeCreatingVertex(EntityMutationContext context, AtlasEntity ent graph.removeVertex(context.getVertex(entity.getGuid())); } + private void resetAllRelationshipAttributes(AtlasEntity entity) { + if (entity.getRemoveRelationshipAttributes() != null) { + entity.setRemoveRelationshipAttributes(null); + } + if (entity.getAppendRelationshipAttributes() != null) { + entity.setAppendRelationshipAttributes(null); + } + if (entity.getRelationshipAttributes() != null) { + entity.setRelationshipAttributes(null); + } + } + private boolean syncContractCertificateStatus(AtlasEntity entity, DataContract contract) throws AtlasBaseException { boolean contractSync = false; // Sync certificateStatus From 418a8cf4db73a93d91cf17068538bf9b0f837990 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Tue, 30 Apr 2024 12:17:18 +0530 Subject: [PATCH 24/25] Ignoring null value fields --- .../v2/preprocessor/contract/DataContract.java | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java index ae45bf2353..dc3cdb466b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java @@ -21,6 +21,7 @@ @JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) @JsonPropertyOrder({"kind", "status", "template_version", "data_source", "dataset", "type", "description", "owners", "tags", "certificate", "columns"}) public class DataContract { @@ -144,10 +145,6 @@ public void setType(String type) throws AtlasBaseException { } } - public void setDescription(String description) { - this.description = description; - } - public void setOwners(List owners) { this.owners = owners; } @@ -156,10 +153,6 @@ public void setTags(List tags) { this.tags = tags; } - public void setCertificate(String certificate) { - this.certificate = certificate; - } - public void setColumns(List columns) { this.columns = columns; } @@ -169,13 +162,13 @@ public void setUnknownFields(String key, Object value) { unknownFields.put(key, value); } - private boolean isSemVer(String version) { Matcher matcher = versionPattern.matcher(version); return matcher.matches(); } @JsonIgnoreProperties(ignoreUnknown = true) + @JsonInclude(JsonInclude.Include.NON_NULL) @JsonPropertyOrder({"name"}) public static final class BusinessTag { @NotNull @@ -194,6 +187,7 @@ public Map getUnknownFields() { } @JsonIgnoreProperties(ignoreUnknown = true) + @JsonInclude(JsonInclude.Include.NON_NULL) @JsonPropertyOrder({"name", "description", "data_type"}) public static final class Column { @NotNull @@ -201,8 +195,6 @@ public static final class Column { public String description; - public boolean is_primary; - public String data_type; private final Map unknownFields = new HashMap<>(); From fb0ee62b5773fc86e5478985c0ddb6b5d8c8780f Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Tue, 30 Apr 2024 18:03:20 +0530 Subject: [PATCH 25/25] Upgrade validator version to avoid vulnerability --- repository/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/pom.xml b/repository/pom.xml index 64cb9c0d48..a2a1a4198f 100755 --- a/repository/pom.xml +++ b/repository/pom.xml @@ -325,7 +325,7 @@ org.hibernate hibernate-validator - 4.3.0.Final + 4.3.2.Final