From 4b00c081920c6f4b65cd96605cef091b0ab59874 Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Fri, 6 Dec 2024 01:20:11 +0530 Subject: [PATCH 01/20] janus0.6 changes --- .github/workflows/maven.yml | 1 + .../org/apache/atlas/AtlasConfiguration.java | 1 + .../store/graph/v2/EntityGraphRetriever.java | 247 +++++++++++++++++- 3 files changed, 245 insertions(+), 4 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index f8a09b5589..e290252038 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,6 +26,7 @@ on: - development - master - lineageondemand + - janus0.6 jobs: build: diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index df2bca7860..60d933d6ae 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -117,6 +117,7 @@ public enum AtlasConfiguration { ATLAS_INDEXSEARCH_QUERY_SIZE_MAX_LIMIT("atlas.indexsearch.query.size.max.limit", 100000), ATLAS_INDEXSEARCH_LIMIT_UTM_TAGS("atlas.indexsearch.limit.ignore.utm.tags", ""), ATLAS_INDEXSEARCH_ENABLE_API_LIMIT("atlas.indexsearch.enable.api.limit", false), + ATLAS_INDEXSEARCH_ENABLE_JANUS_OPTIMISATION("atlas.indexsearch.enable.janus.optimization", false), ATLAS_MAINTENANCE_MODE("atlas.maintenance.mode", false), diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 34d3036042..d3762b04c2 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -49,6 +49,8 @@ import org.apache.atlas.repository.graphdb.AtlasElement; import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.repository.graphdb.janus.AtlasJanusEdge; +import org.apache.atlas.repository.util.AccessControlUtils; import org.apache.atlas.type.AtlasArrayType; import org.apache.atlas.type.AtlasBuiltInTypes.AtlasObjectIdType; import org.apache.atlas.type.AtlasEntityType; @@ -65,8 +67,12 @@ import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.atlas.v1.model.instance.Id; import org.apache.commons.collections.CollectionUtils; +import org.apache.commons.collections.IteratorUtils; import org.apache.commons.collections.MapUtils; import org.apache.commons.lang3.StringUtils; +import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; +import org.apache.tinkerpop.gremlin.structure.Vertex; +import org.apache.tinkerpop.gremlin.structure.VertexProperty; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -88,10 +94,7 @@ import java.util.Objects; import java.util.Queue; import java.util.Set; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.ThreadFactory; +import java.util.concurrent.*; import java.util.stream.Collectors; import static org.apache.atlas.glossary.GlossaryUtils.TERM_ASSIGNMENT_ATTR_CONFIDENCE; @@ -1009,7 +1012,114 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex) return mapVertexToAtlasEntityHeader(entityVertex, Collections.emptySet()); } + private Map preloadProperties(AtlasVertex entityVertex, AtlasEntityType entityType, Set attributes) { + Map propertiesMap = new HashMap<>(); + if (entityType == null) { + return propertiesMap; + } + // Execute the traversal to fetch properties + GraphTraversal> traversal = graph.V(entityVertex.getId()).properties(); + + // Check if any attribute is a struct or object type for edge lookup + boolean isAnyAttributeAStructOrObject = attributes.stream().anyMatch(a -> { + + AtlasAttribute attribute = entityType.getAttribute(a); + if (attribute == null || attribute.getAttributeType() == null) { + return false; + } + + if (!(attribute.getAttributeType() instanceof AtlasArrayType)) { + return false; + + } + AtlasArrayType arrayType = (AtlasArrayType) attribute.getAttributeType(); + + if (arrayType.getElementType() == null) { + return false; + } + + AtlasType arrayElementType = arrayType.getElementType(); + + return arrayElementType.getTypeCategory() == TypeCategory.STRUCT; + }); + + if (isAnyAttributeAStructOrObject) { + List edgeProperties = IteratorUtils.toList(entityVertex.getEdges(AtlasEdgeDirection.OUT).iterator()); + List edgeLabels = + edgeProperties.stream() + .map(e -> { + AtlasJanusEdge edge = (AtlasJanusEdge) e; + return edge.getLabel().toString().substring(2); + }) + .collect(Collectors.toList()); + + edgeLabels.stream().forEach(e -> propertiesMap.put(e, StringUtils.SPACE)); + } + + // Iterate through the resulting VertexProperty objects + while (traversal.hasNext()) { + try { + VertexProperty property = traversal.next(); + + AtlasAttribute attribute = entityType.getAttribute(property.key()); + TypeCategory typeCategory = attribute != null ? attribute.getAttributeType().getTypeCategory() : null; + TypeCategory elementTypeCategory = attribute != null && attribute.getAttributeType().getTypeCategory() == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; + + if (property.isPresent()) { + if (typeCategory == TypeCategory.ARRAY && elementTypeCategory == TypeCategory.PRIMITIVE) { + Object value = propertiesMap.get(property.key()); + if (value instanceof List) { + ((List) value).add(property.value()); + } else { + List values = new ArrayList<>(); + values.add(property.value()); + propertiesMap.put(property.key(), values); + } + } else { + if (propertiesMap.get(property.key()) == null) { + propertiesMap.put(property.key(), property.value()); + } + } + } + } catch (RuntimeException e) { + LOG.error("Error preloading properties for entity vertex: {}", entityVertex, e); + throw e; // Re-throw the exception after logging it + } + } + return propertiesMap; + } + + + private boolean isPolicyAttribute(Set attributes) { + Set exclusionSet = new HashSet<>(Arrays.asList(AccessControlUtils.ATTR_POLICY_TYPE, + AccessControlUtils.ATTR_POLICY_USERS, + AccessControlUtils.ATTR_POLICY_GROUPS, + AccessControlUtils.ATTR_POLICY_ROLES, + AccessControlUtils.ATTR_POLICY_ACTIONS, + AccessControlUtils.ATTR_POLICY_CATEGORY, + AccessControlUtils.ATTR_POLICY_SUB_CATEGORY, + AccessControlUtils.ATTR_POLICY_RESOURCES, + AccessControlUtils.ATTR_POLICY_IS_ENABLED, + AccessControlUtils.ATTR_POLICY_RESOURCES_CATEGORY, + AccessControlUtils.ATTR_POLICY_SERVICE_NAME, + AccessControlUtils.ATTR_POLICY_PRIORITY, + AccessControlUtils.REL_ATTR_POLICIES)); + + return exclusionSet.stream().anyMatch(attributes::contains); + } + private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, Set attributes) throws AtlasBaseException { + boolean shouldPrefetch = !isPolicyAttribute(attributes) + && AtlasConfiguration.ATLAS_INDEXSEARCH_ENABLE_JANUS_OPTIMISATION.getBoolean(); + + if (shouldPrefetch) { + return mapVertexToAtlasEntityHeaderWithPrefetch(entityVertex, attributes); + } else { + return mapVertexToAtlasEntityHeaderWithoutPrefetch(entityVertex, attributes); + } + } + + private AtlasEntityHeader mapVertexToAtlasEntityHeaderWithoutPrefetch(AtlasVertex entityVertex, Set attributes) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("mapVertexToAtlasEntityHeader"); AtlasEntityHeader ret = new AtlasEntityHeader(); try { @@ -1078,6 +1188,7 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, } } + Object attrValue = getVertexAttribute(entityVertex, attribute); if (attrValue != null) { @@ -1093,6 +1204,104 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex, return ret; } + private AtlasEntityHeader mapVertexToAtlasEntityHeaderWithPrefetch(AtlasVertex entityVertex, Set attributes) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("mapVertexToAtlasEntityHeaderWithPrefetch"); + AtlasEntityHeader ret = new AtlasEntityHeader(); + try { + //pre-fetching the properties + String typeName = entityVertex.getProperty(Constants.TYPE_NAME_PROPERTY_KEY, String.class); //properties.get returns null + AtlasEntityType entityType = typeRegistry.getEntityTypeByName(typeName); // this is not costly + Map properties = preloadProperties(entityVertex, entityType, attributes); + + String guid = (String) properties.get(Constants.GUID_PROPERTY_KEY); + + Integer value = (Integer)properties.get(Constants.IS_INCOMPLETE_PROPERTY_KEY); + Boolean isIncomplete = value != null && value.equals(INCOMPLETE_ENTITY_VALUE) ? Boolean.TRUE : Boolean.FALSE; + + ret.setTypeName(typeName); + ret.setGuid(guid); + + String state = (String)properties.get(Constants.STATE_PROPERTY_KEY); + Id.EntityState entityState = state == null ? null : Id.EntityState.valueOf(state); + ret.setStatus((entityState == Id.EntityState.DELETED) ? AtlasEntity.Status.DELETED : AtlasEntity.Status.ACTIVE); + + RequestContext context = RequestContext.get(); + boolean includeClassifications = context.includeClassifications(); + boolean includeClassificationNames = context.isIncludeClassificationNames(); + if(includeClassifications){ + ret.setClassificationNames(getAllTraitNamesFromAttribute(entityVertex)); + } else if (!includeClassifications && includeClassificationNames) { + ret.setClassificationNames(getAllTraitNamesFromAttribute(entityVertex)); + } + ret.setIsIncomplete(isIncomplete); + ret.setLabels(getLabels(entityVertex)); + + ret.setCreatedBy(properties.get(CREATED_BY_KEY) != null ? (String) properties.get(CREATED_BY_KEY) : null); + ret.setUpdatedBy(properties.get(MODIFIED_BY_KEY) != null ? (String) properties.get(MODIFIED_BY_KEY) : null); + ret.setCreateTime(properties.get(TIMESTAMP_PROPERTY_KEY) != null ? new Date((Long)properties.get(TIMESTAMP_PROPERTY_KEY)) : null); + ret.setUpdateTime(properties.get(MODIFICATION_TIMESTAMP_PROPERTY_KEY) != null ? new Date((Long)properties.get(MODIFICATION_TIMESTAMP_PROPERTY_KEY)) : null); + + if(RequestContext.get().includeMeanings()) { + List termAssignmentHeaders = mapAssignedTerms(entityVertex); + ret.setMeanings(termAssignmentHeaders); + ret.setMeaningNames( + termAssignmentHeaders.stream().map(AtlasTermAssignmentHeader::getDisplayText) + .collect(Collectors.toList())); + } + + if (entityType != null) { + for (AtlasAttribute headerAttribute : entityType.getHeaderAttributes().values()) { + Object attrValue = getVertexAttributePreFetchCache(entityVertex, headerAttribute, properties); + + if (attrValue != null) { + ret.setAttribute(headerAttribute.getName(), attrValue); + } + } + + if(properties.get(NAME) != null){ + ret.setDisplayText(properties.get(NAME).toString()); + } else if(properties.get(DISPLAY_NAME) != null) { + ret.setDisplayText(properties.get(DISPLAY_NAME).toString()); + } else if(properties.get(QUALIFIED_NAME) != null) { + ret.setDisplayText(properties.get(QUALIFIED_NAME).toString()); + } + + + + //attributes = only the attributes of entityType + if (CollectionUtils.isNotEmpty(attributes)) { + for (String attrName : attributes) { + AtlasAttribute attribute = entityType.getAttribute(attrName); + + if (attribute == null) { + attrName = toNonQualifiedName(attrName); + + if (ret.hasAttribute(attrName)) { + continue; + } + + attribute = entityType.getAttribute(attrName); + + if (attribute == null) { + attribute = entityType.getRelationshipAttribute(attrName, null); + } + } + + //this is a call to cassandra + Object attrValue = getVertexAttributePreFetchCache(entityVertex, attribute, properties); //use prefetch cache + + if (attrValue != null) { + ret.setAttribute(attrName, attrValue); + } + } + } + } + } finally { + RequestContext.get().endMetricRecord(metricRecorder); + } + return ret; + } + private String toNonQualifiedName(String attrName) { String ret; if (attrName.contains(".")) { @@ -1709,6 +1918,36 @@ public Object getVertexAttribute(AtlasVertex vertex, AtlasAttribute attribute) t return vertex != null && attribute != null ? mapVertexToAttribute(vertex, attribute, null, false) : null; } + public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute attribute, Map properties) throws AtlasBaseException { + if (vertex == null || attribute == null) { + return null; + } + + TypeCategory typeCategory = attribute.getAttributeType().getTypeCategory(); + TypeCategory elementTypeCategory = typeCategory == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; + + // value is present and value is not marker (SPACE for further lookup) and type is primitive or array of primitives + if (properties.get(attribute.getName()) != null && properties.get(attribute.getName()) != StringUtils.SPACE && + (typeCategory.equals(TypeCategory.PRIMITIVE) || (elementTypeCategory == null || elementTypeCategory.equals(TypeCategory.PRIMITIVE)))) { + return properties.get(attribute.getName()); + } + + // if value is empty && element is array of primitives, return empty list + if (properties.get(attribute.getName()) == null && (elementTypeCategory == null || elementTypeCategory.equals(TypeCategory.PRIMITIVE))) { + return new ArrayList<>(); + } + + // value is present as marker, fetch the value from the vertex + if (properties.get(attribute.getName()) != null) { + AtlasPerfMetrics.MetricRecorder nonPrimitiveAttributes = RequestContext.get().startMetricRecord("processNonPrimitiveAttributes"); + Object mappedVertex = mapVertexToAttribute(vertex, attribute, null, false); + RequestContext.get().endMetricRecord(nonPrimitiveAttributes); + return mappedVertex; + } + + return null; + } + private Object getVertexAttributeIgnoreInactive(AtlasVertex vertex, AtlasAttribute attribute) throws AtlasBaseException { return vertex != null && attribute != null ? mapVertexToAttribute(vertex, attribute, null, false, true, true) : null; } From af722cba4e49c49178ec8f0046b4dc0d036e1a4f Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Fri, 6 Dec 2024 01:51:55 +0530 Subject: [PATCH 02/20] fix object type and simplify conditional logic --- .../store/graph/v2/EntityGraphRetriever.java | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index d3762b04c2..cc22257f13 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1014,23 +1014,26 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeader(AtlasVertex entityVertex) private Map preloadProperties(AtlasVertex entityVertex, AtlasEntityType entityType, Set attributes) { Map propertiesMap = new HashMap<>(); - if (entityType == null) { - return propertiesMap; - } + // Execute the traversal to fetch properties GraphTraversal> traversal = graph.V(entityVertex.getId()).properties(); // Check if any attribute is a struct or object type for edge lookup boolean isAnyAttributeAStructOrObject = attributes.stream().anyMatch(a -> { - + if (entityType == null) { + return false; + } AtlasAttribute attribute = entityType.getAttribute(a); if (attribute == null || attribute.getAttributeType() == null) { return false; } + if ((attribute.getAttributeType() instanceof AtlasStructType)) { + return true; + } + if (!(attribute.getAttributeType() instanceof AtlasArrayType)) { return false; - } AtlasArrayType arrayType = (AtlasArrayType) attribute.getAttributeType(); @@ -1925,15 +1928,16 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute TypeCategory typeCategory = attribute.getAttributeType().getTypeCategory(); TypeCategory elementTypeCategory = typeCategory == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; + boolean isArrayOfPrimitives = typeCategory.equals(TypeCategory.ARRAY) && elementTypeCategory.equals(TypeCategory.PRIMITIVE); // value is present and value is not marker (SPACE for further lookup) and type is primitive or array of primitives if (properties.get(attribute.getName()) != null && properties.get(attribute.getName()) != StringUtils.SPACE && - (typeCategory.equals(TypeCategory.PRIMITIVE) || (elementTypeCategory == null || elementTypeCategory.equals(TypeCategory.PRIMITIVE)))) { + (typeCategory.equals(TypeCategory.PRIMITIVE) || isArrayOfPrimitives)) { return properties.get(attribute.getName()); } // if value is empty && element is array of primitives, return empty list - if (properties.get(attribute.getName()) == null && (elementTypeCategory == null || elementTypeCategory.equals(TypeCategory.PRIMITIVE))) { + if (properties.get(attribute.getName()) == null && isArrayOfPrimitives) { return new ArrayList<>(); } From 4f1289b53e3b4c4c762d5f0bdd447a6de0c58818 Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Fri, 6 Dec 2024 02:23:09 +0530 Subject: [PATCH 03/20] better approach --- .../atlas/repository/store/graph/v2/EntityGraphRetriever.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index cc22257f13..40cdac6a2f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1052,7 +1052,8 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt edgeProperties.stream() .map(e -> { AtlasJanusEdge edge = (AtlasJanusEdge) e; - return edge.getLabel().toString().substring(2); + String edgeLabel = edge.getLabel(); + return edgeLabel.replaceFirst(EDGE_LABEL_PREFIX , StringUtils.EMPTY); }) .collect(Collectors.toList()); From 74858aa62f3d573d0533560172e51a97c40c364d Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Fri, 6 Dec 2024 03:13:06 +0530 Subject: [PATCH 04/20] fallback always --- .../main/java/org/apache/atlas/AtlasConfiguration.java | 4 ++++ .../repository/store/graph/v2/EntityGraphRetriever.java | 8 +++----- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index 60d933d6ae..b9c3c338c2 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -118,6 +118,10 @@ public enum AtlasConfiguration { ATLAS_INDEXSEARCH_LIMIT_UTM_TAGS("atlas.indexsearch.limit.ignore.utm.tags", ""), ATLAS_INDEXSEARCH_ENABLE_API_LIMIT("atlas.indexsearch.enable.api.limit", false), ATLAS_INDEXSEARCH_ENABLE_JANUS_OPTIMISATION("atlas.indexsearch.enable.janus.optimization", false), + /*** + * This configuration is used to enable fetching non primitive attributes in index search + */ + ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES("atlas.indexsearch.enable.fetching.non.primitive.attributes", true), ATLAS_MAINTENANCE_MODE("atlas.maintenance.mode", false), diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 40cdac6a2f..bc0f7f5291 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -97,6 +97,7 @@ import java.util.concurrent.*; import java.util.stream.Collectors; +import static org.apache.atlas.AtlasConfiguration.ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES; import static org.apache.atlas.glossary.GlossaryUtils.TERM_ASSIGNMENT_ATTR_CONFIDENCE; import static org.apache.atlas.glossary.GlossaryUtils.TERM_ASSIGNMENT_ATTR_CREATED_BY; import static org.apache.atlas.glossary.GlossaryUtils.TERM_ASSIGNMENT_ATTR_DESCRIPTION; @@ -1943,11 +1944,8 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute } // value is present as marker, fetch the value from the vertex - if (properties.get(attribute.getName()) != null) { - AtlasPerfMetrics.MetricRecorder nonPrimitiveAttributes = RequestContext.get().startMetricRecord("processNonPrimitiveAttributes"); - Object mappedVertex = mapVertexToAttribute(vertex, attribute, null, false); - RequestContext.get().endMetricRecord(nonPrimitiveAttributes); - return mappedVertex; + if (ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES.getBoolean()) { + return mapVertexToAttribute(vertex, attribute, null, false); } return null; From d237e9aa0481855c45016ce1250d6b2daa631d40 Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Fri, 6 Dec 2024 09:42:21 +0530 Subject: [PATCH 05/20] add log to track non null attributes --- .../repository/store/graph/v2/EntityGraphRetriever.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index bc0f7f5291..81e7044ce4 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1945,7 +1945,11 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute // value is present as marker, fetch the value from the vertex if (ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES.getBoolean()) { - return mapVertexToAttribute(vertex, attribute, null, false); + AtlasPerfMetrics.MetricRecorder nonPrimitiveAttributes = RequestContext.get().startMetricRecord("processNonPrimitiveAttributes"); + Object mappedVertex = mapVertexToAttribute(vertex, attribute, null, false); + LOG.debug("capturing excluded property set category and value, mapVertexValue - {}: {} : {} : {}", attribute.getName(), attribute.getAttributeType().getTypeCategory(), properties.get(attribute.getName()), mappedVertex); + RequestContext.get().endMetricRecord(nonPrimitiveAttributes); + return mappedVertex; } return null; From d1e72e0e2df9bd2d2700812541fecd2c91a8df8b Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Fri, 6 Dec 2024 10:26:05 +0530 Subject: [PATCH 06/20] add log to understand relationship labels --- .../repository/store/graph/v2/EntityGraphRetriever.java | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 81e7044ce4..2aa13381f7 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1054,6 +1054,10 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt .map(e -> { AtlasJanusEdge edge = (AtlasJanusEdge) e; String edgeLabel = edge.getLabel(); + // check if edgeLabel contains __ as prefix + if(!edgeLabel.contains(EDGE_LABEL_PREFIX)) { + LOG.debug("Edge label {} does not contain prefix", edgeLabel); + } return edgeLabel.replaceFirst(EDGE_LABEL_PREFIX , StringUtils.EMPTY); }) .collect(Collectors.toList()); @@ -1938,6 +1942,11 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute return properties.get(attribute.getName()); } + //when value is not present and type is primitive, return null + if(properties.get(attribute.getName()) == null && typeCategory.equals(TypeCategory.PRIMITIVE)) { + return null; + } + // if value is empty && element is array of primitives, return empty list if (properties.get(attribute.getName()) == null && isArrayOfPrimitives) { return new ArrayList<>(); From 837d50785ffd61affb30b31f45fafc4c658567bc Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Fri, 6 Dec 2024 10:40:45 +0530 Subject: [PATCH 07/20] use prefetch final for map,struct and array of primitives --- .../repository/store/graph/v2/EntityGraphRetriever.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 2aa13381f7..407493ab95 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1935,15 +1935,15 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute TypeCategory typeCategory = attribute.getAttributeType().getTypeCategory(); TypeCategory elementTypeCategory = typeCategory == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; boolean isArrayOfPrimitives = typeCategory.equals(TypeCategory.ARRAY) && elementTypeCategory.equals(TypeCategory.PRIMITIVE); + boolean isPrefetchValueFinal = (typeCategory.equals(TypeCategory.PRIMITIVE) || typeCategory.equals(TypeCategory.ENUM) || typeCategory.equals(TypeCategory.MAP) || isArrayOfPrimitives); // value is present and value is not marker (SPACE for further lookup) and type is primitive or array of primitives - if (properties.get(attribute.getName()) != null && properties.get(attribute.getName()) != StringUtils.SPACE && - (typeCategory.equals(TypeCategory.PRIMITIVE) || isArrayOfPrimitives)) { + if (properties.get(attribute.getName()) != null && properties.get(attribute.getName()) != StringUtils.SPACE && isPrefetchValueFinal) { return properties.get(attribute.getName()); } //when value is not present and type is primitive, return null - if(properties.get(attribute.getName()) == null && typeCategory.equals(TypeCategory.PRIMITIVE)) { + if(properties.get(attribute.getName()) == null && isPrefetchValueFinal) { return null; } From 5c5bb2d3d293d03cc11bf18fd7e0a32bfa70e1b0 Mon Sep 17 00:00:00 2001 From: sriram-atlan Date: Fri, 6 Dec 2024 11:20:12 +0530 Subject: [PATCH 08/20] fallback on relationship links only --- .../store/graph/v2/EntityGraphRetriever.java | 57 ++----------------- 1 file changed, 6 insertions(+), 51 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 407493ab95..88bc1348b0 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1019,52 +1019,6 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt // Execute the traversal to fetch properties GraphTraversal> traversal = graph.V(entityVertex.getId()).properties(); - // Check if any attribute is a struct or object type for edge lookup - boolean isAnyAttributeAStructOrObject = attributes.stream().anyMatch(a -> { - if (entityType == null) { - return false; - } - AtlasAttribute attribute = entityType.getAttribute(a); - if (attribute == null || attribute.getAttributeType() == null) { - return false; - } - - if ((attribute.getAttributeType() instanceof AtlasStructType)) { - return true; - } - - if (!(attribute.getAttributeType() instanceof AtlasArrayType)) { - return false; - } - AtlasArrayType arrayType = (AtlasArrayType) attribute.getAttributeType(); - - if (arrayType.getElementType() == null) { - return false; - } - - AtlasType arrayElementType = arrayType.getElementType(); - - return arrayElementType.getTypeCategory() == TypeCategory.STRUCT; - }); - - if (isAnyAttributeAStructOrObject) { - List edgeProperties = IteratorUtils.toList(entityVertex.getEdges(AtlasEdgeDirection.OUT).iterator()); - List edgeLabels = - edgeProperties.stream() - .map(e -> { - AtlasJanusEdge edge = (AtlasJanusEdge) e; - String edgeLabel = edge.getLabel(); - // check if edgeLabel contains __ as prefix - if(!edgeLabel.contains(EDGE_LABEL_PREFIX)) { - LOG.debug("Edge label {} does not contain prefix", edgeLabel); - } - return edgeLabel.replaceFirst(EDGE_LABEL_PREFIX , StringUtils.EMPTY); - }) - .collect(Collectors.toList()); - - edgeLabels.stream().forEach(e -> propertiesMap.put(e, StringUtils.SPACE)); - } - // Iterate through the resulting VertexProperty objects while (traversal.hasNext()) { try { @@ -1292,6 +1246,7 @@ private AtlasEntityHeader mapVertexToAtlasEntityHeaderWithPrefetch(AtlasVertex e attribute = entityType.getAttribute(attrName); if (attribute == null) { + // dataContractLatest, meanings, links attribute = entityType.getRelationshipAttribute(attrName, null); } } @@ -1954,11 +1909,11 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute // value is present as marker, fetch the value from the vertex if (ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES.getBoolean()) { - AtlasPerfMetrics.MetricRecorder nonPrimitiveAttributes = RequestContext.get().startMetricRecord("processNonPrimitiveAttributes"); - Object mappedVertex = mapVertexToAttribute(vertex, attribute, null, false); - LOG.debug("capturing excluded property set category and value, mapVertexValue - {}: {} : {} : {}", attribute.getName(), attribute.getAttributeType().getTypeCategory(), properties.get(attribute.getName()), mappedVertex); - RequestContext.get().endMetricRecord(nonPrimitiveAttributes); - return mappedVertex; + //AtlasPerfMetrics.MetricRecorder nonPrimitiveAttributes = RequestContext.get().startMetricRecord("processNonPrimitiveAttributes"); + return mapVertexToAttribute(vertex, attribute, null, false); + //LOG.debug("capturing excluded property set category and value, mapVertexValue - {}: {} : {} : {}", attribute.getName(), attribute.getAttributeType().getTypeCategory(), properties.get(attribute.getName()), mappedVertex); + //RequestContext.get().endMetricRecord(nonPrimitiveAttributes); + //return mappedVertex; } return null; From 62c4f9712397f5733abe666505da16fbcc52999b Mon Sep 17 00:00:00 2001 From: aarshi Date: Fri, 6 Dec 2024 14:22:30 +0530 Subject: [PATCH 09/20] preload properties for relatedvertex --- .../repository/store/graph/v2/EntityGraphRetriever.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 88bc1348b0..78666f4cdc 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -310,12 +310,16 @@ public AtlasObjectId toAtlasObjectId(AtlasVertex entityVertex) throws AtlasBaseE Map attributes = new HashMap<>(); Set relationAttributes = RequestContext.get().getRelationAttrsForSearch(); + + // preloadProperties here if (CollectionUtils.isNotEmpty(relationAttributes)) { + Map referenceVertexProperties = preloadProperties(entityVertex, entityType, Collections.emptySet()); + for (String attributeName : relationAttributes) { AtlasAttribute attribute = entityType.getAttribute(attributeName); if (attribute != null && !uniqueAttributes.containsKey(attributeName)) { - Object attrValue = getVertexAttribute(entityVertex, attribute); + Object attrValue = getVertexAttributePreFetchCache(entityVertex, attribute, referenceVertexProperties); if (attrValue != null) { attributes.put(attribute.getName(), attrValue); } From bf345392949e582f7e5f6dc4e5c03aefa10844d0 Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 9 Dec 2024 10:50:40 +0530 Subject: [PATCH 10/20] remove check for maps --- .../atlas/repository/store/graph/v2/EntityGraphRetriever.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 78666f4cdc..4f6ec87933 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1894,7 +1894,7 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute TypeCategory typeCategory = attribute.getAttributeType().getTypeCategory(); TypeCategory elementTypeCategory = typeCategory == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; boolean isArrayOfPrimitives = typeCategory.equals(TypeCategory.ARRAY) && elementTypeCategory.equals(TypeCategory.PRIMITIVE); - boolean isPrefetchValueFinal = (typeCategory.equals(TypeCategory.PRIMITIVE) || typeCategory.equals(TypeCategory.ENUM) || typeCategory.equals(TypeCategory.MAP) || isArrayOfPrimitives); + boolean isPrefetchValueFinal = (typeCategory.equals(TypeCategory.PRIMITIVE) || typeCategory.equals(TypeCategory.ENUM) || isArrayOfPrimitives); // value is present and value is not marker (SPACE for further lookup) and type is primitive or array of primitives if (properties.get(attribute.getName()) != null && properties.get(attribute.getName()) != StringUtils.SPACE && isPrefetchValueFinal) { From f6c9bde646f2314b59b8c04138f46e884705811f Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 9 Dec 2024 14:22:55 +0530 Subject: [PATCH 11/20] fix business attributes --- .../store/graph/v2/EntityGraphRetriever.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 4f6ec87933..91628b6cd5 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1031,9 +1031,10 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt AtlasAttribute attribute = entityType.getAttribute(property.key()); TypeCategory typeCategory = attribute != null ? attribute.getAttributeType().getTypeCategory() : null; TypeCategory elementTypeCategory = attribute != null && attribute.getAttributeType().getTypeCategory() == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; + boolean isBusinessAttribute = attribute == null; if (property.isPresent()) { - if (typeCategory == TypeCategory.ARRAY && elementTypeCategory == TypeCategory.PRIMITIVE) { + if ((typeCategory == TypeCategory.ARRAY && elementTypeCategory == TypeCategory.PRIMITIVE)) { Object value = propertiesMap.get(property.key()); if (value instanceof List) { ((List) value).add(property.value()); @@ -1045,6 +1046,12 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt } else { if (propertiesMap.get(property.key()) == null) { propertiesMap.put(property.key(), property.value()); + } else if (isBusinessAttribute) { // If it is a business attribute, and is a multi-valued attribute + LOG.info("Duplicate property key {} found for entity vertex: {}", property.key(), entityVertex); + List values = new ArrayList<>(); + values.add(propertiesMap.get(property.key())); + values.add(property.value()); + propertiesMap.put(property.key(), values); } } } @@ -1891,10 +1898,11 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute return null; } + TypeCategory typeCategory = attribute.getAttributeType().getTypeCategory(); TypeCategory elementTypeCategory = typeCategory == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; boolean isArrayOfPrimitives = typeCategory.equals(TypeCategory.ARRAY) && elementTypeCategory.equals(TypeCategory.PRIMITIVE); - boolean isPrefetchValueFinal = (typeCategory.equals(TypeCategory.PRIMITIVE) || typeCategory.equals(TypeCategory.ENUM) || isArrayOfPrimitives); + boolean isPrefetchValueFinal = (typeCategory.equals(TypeCategory.PRIMITIVE) || typeCategory.equals(TypeCategory.ENUM) || typeCategory.equals(TypeCategory.MAP) || isArrayOfPrimitives); // value is present and value is not marker (SPACE for further lookup) and type is primitive or array of primitives if (properties.get(attribute.getName()) != null && properties.get(attribute.getName()) != StringUtils.SPACE && isPrefetchValueFinal) { From 664501fda8d6d200d4e66b3321fec57d09e394f6 Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 9 Dec 2024 14:25:58 +0530 Subject: [PATCH 12/20] fix log level --- .../atlas/repository/store/graph/v2/EntityGraphRetriever.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 91628b6cd5..d1b2aef6be 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1034,7 +1034,7 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt boolean isBusinessAttribute = attribute == null; if (property.isPresent()) { - if ((typeCategory == TypeCategory.ARRAY && elementTypeCategory == TypeCategory.PRIMITIVE)) { + if (typeCategory == TypeCategory.ARRAY && elementTypeCategory == TypeCategory.PRIMITIVE) { Object value = propertiesMap.get(property.key()); if (value instanceof List) { ((List) value).add(property.value()); @@ -1047,7 +1047,7 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt if (propertiesMap.get(property.key()) == null) { propertiesMap.put(property.key(), property.value()); } else if (isBusinessAttribute) { // If it is a business attribute, and is a multi-valued attribute - LOG.info("Duplicate property key {} found for entity vertex: {}", property.key(), entityVertex); + LOG.warn("Duplicate property key {} found for entity vertex: {}", property.key(), entityVertex); List values = new ArrayList<>(); values.add(propertiesMap.get(property.key())); values.add(property.value()); From 87568edfee9559207a13038c790cccc66facfb85 Mon Sep 17 00:00:00 2001 From: aarshi Date: Mon, 9 Dec 2024 17:30:10 +0530 Subject: [PATCH 13/20] update branch --- .github/workflows/maven.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index e290252038..f8a09b5589 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,7 +26,6 @@ on: - development - master - lineageondemand - - janus0.6 jobs: build: From 291cb2e09d47bffb0f57fd90430921e24368da16 Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 10 Dec 2024 13:52:08 +0530 Subject: [PATCH 14/20] add strict check for business attribute --- .../store/graph/v2/EntityGraphRetriever.java | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index d1b2aef6be..a4b0916258 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1031,7 +1031,7 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt AtlasAttribute attribute = entityType.getAttribute(property.key()); TypeCategory typeCategory = attribute != null ? attribute.getAttributeType().getTypeCategory() : null; TypeCategory elementTypeCategory = attribute != null && attribute.getAttributeType().getTypeCategory() == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; - boolean isBusinessAttribute = attribute == null; + boolean canBeBusinessAttribute = attribute == null; if (property.isPresent()) { if (typeCategory == TypeCategory.ARRAY && elementTypeCategory == TypeCategory.PRIMITIVE) { @@ -1046,7 +1046,7 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt } else { if (propertiesMap.get(property.key()) == null) { propertiesMap.put(property.key(), property.value()); - } else if (isBusinessAttribute) { // If it is a business attribute, and is a multi-valued attribute + } else if (canBeBusinessAttribute) { // If it is a business attribute, and is a multi-valued attribute LOG.warn("Duplicate property key {} found for entity vertex: {}", property.key(), entityVertex); List values = new ArrayList<>(); values.add(propertiesMap.get(property.key())); @@ -1903,22 +1903,23 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute TypeCategory elementTypeCategory = typeCategory == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; boolean isArrayOfPrimitives = typeCategory.equals(TypeCategory.ARRAY) && elementTypeCategory.equals(TypeCategory.PRIMITIVE); boolean isPrefetchValueFinal = (typeCategory.equals(TypeCategory.PRIMITIVE) || typeCategory.equals(TypeCategory.ENUM) || typeCategory.equals(TypeCategory.MAP) || isArrayOfPrimitives); + boolean isMultiValueBusinessAttribute = attribute.getDefinedInType() != null && attribute.getDefinedInType().getTypeCategory() == TypeCategory.BUSINESS_METADATA && isArrayOfPrimitives; // value is present and value is not marker (SPACE for further lookup) and type is primitive or array of primitives - if (properties.get(attribute.getName()) != null && properties.get(attribute.getName()) != StringUtils.SPACE && isPrefetchValueFinal) { + if (properties.get(attribute.getName()) != null && properties.get(attribute.getName()) != StringUtils.SPACE && (isMultiValueBusinessAttribute || isPrefetchValueFinal)) { return properties.get(attribute.getName()); } - //when value is not present and type is primitive, return null - if(properties.get(attribute.getName()) == null && isPrefetchValueFinal) { - return null; - } - // if value is empty && element is array of primitives, return empty list if (properties.get(attribute.getName()) == null && isArrayOfPrimitives) { return new ArrayList<>(); } + //when value is not present and type is primitive, return null + if(properties.get(attribute.getName()) == null && isPrefetchValueFinal) { + return null; + } + // value is present as marker, fetch the value from the vertex if (ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES.getBoolean()) { //AtlasPerfMetrics.MetricRecorder nonPrimitiveAttributes = RequestContext.get().startMetricRecord("processNonPrimitiveAttributes"); From 10781824bb0d7953bf554d3f3e3d8f5f63269326 Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 10 Dec 2024 16:27:22 +0530 Subject: [PATCH 15/20] code cleanup --- .../store/graph/v2/EntityGraphRetriever.java | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index a4b0916258..e8df762193 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -73,6 +73,8 @@ import org.apache.tinkerpop.gremlin.process.traversal.dsl.graph.GraphTraversal; import org.apache.tinkerpop.gremlin.structure.Vertex; import org.apache.tinkerpop.gremlin.structure.VertexProperty; +import org.janusgraph.core.Cardinality; +import org.janusgraph.graphdb.relations.CacheVertexProperty; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.stereotype.Component; @@ -1031,28 +1033,16 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt AtlasAttribute attribute = entityType.getAttribute(property.key()); TypeCategory typeCategory = attribute != null ? attribute.getAttributeType().getTypeCategory() : null; TypeCategory elementTypeCategory = attribute != null && attribute.getAttributeType().getTypeCategory() == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; - boolean canBeBusinessAttribute = attribute == null; + boolean isMultiValuedProperty = (property instanceof CacheVertexProperty && ((CacheVertexProperty) property).propertyKey().cardinality().equals(Cardinality.SET)); if (property.isPresent()) { if (typeCategory == TypeCategory.ARRAY && elementTypeCategory == TypeCategory.PRIMITIVE) { - Object value = propertiesMap.get(property.key()); - if (value instanceof List) { - ((List) value).add(property.value()); - } else { - List values = new ArrayList<>(); - values.add(property.value()); - propertiesMap.put(property.key(), values); - } - } else { - if (propertiesMap.get(property.key()) == null) { - propertiesMap.put(property.key(), property.value()); - } else if (canBeBusinessAttribute) { // If it is a business attribute, and is a multi-valued attribute - LOG.warn("Duplicate property key {} found for entity vertex: {}", property.key(), entityVertex); - List values = new ArrayList<>(); - values.add(propertiesMap.get(property.key())); - values.add(property.value()); - propertiesMap.put(property.key(), values); - } + updateAttrValue(propertiesMap, property); + } else if (attribute == null && isMultiValuedProperty) { + LOG.warn("Multiple value found for key {} for entity vertex: {}", property.key(), entityVertex); + updateAttrValue(propertiesMap, property); + } else if (propertiesMap.get(property.key()) == null) { + propertiesMap.put(property.key(), property.value()); } } } catch (RuntimeException e) { @@ -1063,6 +1053,16 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt return propertiesMap; } + private void updateAttrValue( Map propertiesMap, VertexProperty property){ + Object value = propertiesMap.get(property.key()); + if (value instanceof List) { + ((List) value).add(property.value()); + } else { + List values = new ArrayList<>(); + values.add(property.value()); + propertiesMap.put(property.key(), values); + } + } private boolean isPolicyAttribute(Set attributes) { Set exclusionSet = new HashSet<>(Arrays.asList(AccessControlUtils.ATTR_POLICY_TYPE, From dce4f05e90fced4d28fb53c248f7645f585b82af Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 10 Dec 2024 17:14:17 +0530 Subject: [PATCH 16/20] compute flag only if property exists --- .../repository/store/graph/v2/EntityGraphRetriever.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index e8df762193..f49d343e32 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1033,9 +1033,13 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt AtlasAttribute attribute = entityType.getAttribute(property.key()); TypeCategory typeCategory = attribute != null ? attribute.getAttributeType().getTypeCategory() : null; TypeCategory elementTypeCategory = attribute != null && attribute.getAttributeType().getTypeCategory() == TypeCategory.ARRAY ? ((AtlasArrayType) attribute.getAttributeType()).getElementType().getTypeCategory() : null; - boolean isMultiValuedProperty = (property instanceof CacheVertexProperty && ((CacheVertexProperty) property).propertyKey().cardinality().equals(Cardinality.SET)); if (property.isPresent()) { + + // If the attribute is not known (null) + // validate if it is multi-valued + boolean isMultiValuedProperty = (property instanceof CacheVertexProperty && ((CacheVertexProperty) property).propertyKey().cardinality().equals(Cardinality.SET)); + if (typeCategory == TypeCategory.ARRAY && elementTypeCategory == TypeCategory.PRIMITIVE) { updateAttrValue(propertiesMap, property); } else if (attribute == null && isMultiValuedProperty) { From b2a58a64dcbf23a0b927695a65ec8f2e881396cc Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 10 Dec 2024 17:15:47 +0530 Subject: [PATCH 17/20] verbose comment --- .../atlas/repository/store/graph/v2/EntityGraphRetriever.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index f49d343e32..09ec1c3eb2 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1037,7 +1037,7 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt if (property.isPresent()) { // If the attribute is not known (null) - // validate if it is multi-valued + // validate if prefetched property is multi-valued boolean isMultiValuedProperty = (property instanceof CacheVertexProperty && ((CacheVertexProperty) property).propertyKey().cardinality().equals(Cardinality.SET)); if (typeCategory == TypeCategory.ARRAY && elementTypeCategory == TypeCategory.PRIMITIVE) { From 0da7195c976c664c3346a7404816ba062e96297b Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 10 Dec 2024 19:23:52 +0530 Subject: [PATCH 18/20] add policy variables --- .../store/graph/v2/EntityGraphRetriever.java | 11 ++++++----- .../atlas/repository/util/AccessControlUtils.java | 4 ++++ 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index 09ec1c3eb2..e9ee72fcfe 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1081,7 +1081,12 @@ private boolean isPolicyAttribute(Set attributes) { AccessControlUtils.ATTR_POLICY_RESOURCES_CATEGORY, AccessControlUtils.ATTR_POLICY_SERVICE_NAME, AccessControlUtils.ATTR_POLICY_PRIORITY, - AccessControlUtils.REL_ATTR_POLICIES)); + AccessControlUtils.REL_ATTR_POLICIES, + AccessControlUtils.ATTR_SERVICE_SERVICE_TYPE, + AccessControlUtils.ATTR_SERVICE_TAG_SERVICE, + AccessControlUtils.ATTR_SERVICE_IS_ENABLED, + AccessControlUtils.ATTR_SERVICE_LAST_SYNC) + ); return exclusionSet.stream().anyMatch(attributes::contains); } @@ -1926,11 +1931,7 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute // value is present as marker, fetch the value from the vertex if (ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES.getBoolean()) { - //AtlasPerfMetrics.MetricRecorder nonPrimitiveAttributes = RequestContext.get().startMetricRecord("processNonPrimitiveAttributes"); return mapVertexToAttribute(vertex, attribute, null, false); - //LOG.debug("capturing excluded property set category and value, mapVertexValue - {}: {} : {} : {}", attribute.getName(), attribute.getAttributeType().getTypeCategory(), properties.get(attribute.getName()), mappedVertex); - //RequestContext.get().endMetricRecord(nonPrimitiveAttributes); - //return mappedVertex; } return null; diff --git a/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java b/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java index 7101868e30..1f89e4d65c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java @@ -61,6 +61,10 @@ public final class AccessControlUtils { public static final String ATTR_PURPOSE_CLASSIFICATIONS = "purposeClassifications"; + public static final String ATTR_SERVICE_SERVICE_TYPE = "authServiceType"; + public static final String ATTR_SERVICE_TAG_SERVICE = "tagService"; + public static final String ATTR_SERVICE_IS_ENABLED = "authServiceIsEnabled"; + public static final String ATTR_SERVICE_LAST_SYNC = "authServicePolicyLastSync"; public static final String ATTR_POLICY_TYPE = "policyType"; public static final String ATTR_POLICY_USERS = "policyUsers"; public static final String ATTR_POLICY_GROUPS = "policyGroups"; From 759c2302f694ae1e3725cacd8743444d896b9225 Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 10 Dec 2024 20:03:55 +0530 Subject: [PATCH 19/20] fix pipeline --- .github/workflows/maven.yml | 36 +++++++++++++++++++++++------------- 1 file changed, 23 insertions(+), 13 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index f8a09b5589..b8481ac283 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -1,3 +1,4 @@ + # # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file @@ -26,6 +27,7 @@ on: - development - master - lineageondemand + - janus0.6 jobs: build: @@ -34,12 +36,12 @@ jobs: steps: - uses: actions/checkout@v2 - + - name: Set up JDK 1.8 uses: actions/setup-java@v1 with: java-version: 1.8 - + - name: Cache Maven packages uses: actions/cache@v2 with: @@ -48,8 +50,9 @@ jobs: restore-keys: ${{ runner.os }}-m2 - name: Get branch name - run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})" - id: get_branch + run: | + echo "BRANCH_NAME=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV + echo BRANCH_NAME=${GITHUB_REF#refs/heads/} - name: Create Maven Settings uses: s4u/maven-settings-action@v2.8.0 @@ -63,7 +66,7 @@ jobs: - name: Build with Maven run: | - branch_name=${{ steps.get_branch.outputs.branch }} + branch_name=${{ env.BRANCH_NAME }} if [[ $branch_name == 'main' || $branch_name == 'master' || $branch_name == 'lineageondemand' ]] then echo "build without dashboard" @@ -73,19 +76,26 @@ jobs: chmod +x ./build.sh && ./build.sh fi - - run: echo "REPOSITORY_NAME=`echo "$GITHUB_REPOSITORY" | awk -F / '{print $2}' | sed -e "s/:refs//"`" >> $GITHUB_ENV + - name: Get Repository Name + run: echo "REPOSITORY_NAME=`echo "$GITHUB_REPOSITORY" | awk -F / '{print $2}' | sed -e "s/:refs//"`" >> $GITHUB_ENV shell: bash - name: Get version tag - run: echo "##[set-output name=version;]$(echo `git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ steps.get_branch.outputs.branch }} | awk '{ print $1}' | cut -c1-7`)abcd" - id: get_version + # run: echo "##[set-output name=version;]$(echo `git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ env.BRANCH_NAME }} | awk '{ print $1}' | cut -c1-7`)abcd" + run: | + echo "VERSION=$(git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ env.BRANCH_NAME }} | awk '{ print $1}' | cut -c1-7 | head -n 1)abcd" + echo "VERSION=$(git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ env.BRANCH_NAME }} | awk '{ print $1}' | cut -c1-7 | tr -d '[:space:]')abcd" + echo "VERSION=$(git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ env.BRANCH_NAME }} | awk '{ print $1}' | cut -c1-7 | tr -d '[:space:]')abcd" >> $GITHUB_ENV + + - name: Get commit ID + run: echo "COMMIT_ID=$(echo ${GITHUB_SHA} | cut -c1-7)abcd" >> $GITHUB_ENV - - name: Set up Buildx + - name: Set up Buildx id: buildx uses: docker/setup-buildx-action@v1 - name: Login to GitHub Registry - uses: docker/login-action@v1 + uses: docker/login-action@v1 with: registry: ghcr.io username: $GITHUB_ACTOR @@ -102,8 +112,8 @@ jobs: provenance: true push: true tags: | - ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch }}:latest - ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch }}:${{ steps.get_version.outputs.version }} + ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ env.BRANCH_NAME }}:latest + ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ env.BRANCH_NAME }}:${{ env.COMMIT_ID }} - name: Scan Image uses: aquasecurity/trivy-action@master @@ -116,4 +126,4 @@ jobs: - name: Upload Trivy scan results to GitHub Security tab uses: github/codeql-action/upload-sarif@v2.1.33 with: - sarif_file: 'trivy-image-results.sarif' + sarif_file: 'trivy-image-results.sarif' \ No newline at end of file From f2de42dafe54c7fd945445d347a03e6da4176103 Mon Sep 17 00:00:00 2001 From: aarshi Date: Tue, 10 Dec 2024 21:00:43 +0530 Subject: [PATCH 20/20] remove log line --- .../atlas/repository/store/graph/v2/EntityGraphRetriever.java | 1 - 1 file changed, 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java index e9ee72fcfe..59125c492c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java @@ -1043,7 +1043,6 @@ private Map preloadProperties(AtlasVertex entityVertex, AtlasEnt if (typeCategory == TypeCategory.ARRAY && elementTypeCategory == TypeCategory.PRIMITIVE) { updateAttrValue(propertiesMap, property); } else if (attribute == null && isMultiValuedProperty) { - LOG.warn("Multiple value found for key {} for entity vertex: {}", property.key(), entityVertex); updateAttrValue(propertiesMap, property); } else if (propertiesMap.get(property.key()) == null) { propertiesMap.put(property.key(), property.value());