From 82db3c9c3c352cd84c497d37136b7ec08a455556 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Tue, 19 Sep 2023 13:59:44 +0530 Subject: [PATCH 1/8] DG-113 New attributes for Persona and Purpose preferences --- addons/models/0000-Area0/0010-base_model.json | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/addons/models/0000-Area0/0010-base_model.json b/addons/models/0000-Area0/0010-base_model.json index 3f1dc7f77f..b3886f4818 100644 --- a/addons/models/0000-Area0/0010-base_model.json +++ b/addons/models/0000-Area0/0010-base_model.json @@ -879,7 +879,7 @@ ], "description": "Atlan Type representing parent model for Persona, Purpose", "serviceType": "atlan", - "typeVersion": "1.1", + "typeVersion": "1.2", "attributeDefs": [ { "name": "isAccessControlEnabled", @@ -914,6 +914,16 @@ "skipScrubbing": true, "includeInNotification": false }, + { + "name": "denyAssetFilters", + "typeName": "array", + "cardinality": "SET", + "isIndexable": false, + "isOptional": true, + "isUnique": false, + "skipScrubbing": true, + "includeInNotification": false + }, { "name": "channelLink", "typeName": "string", From dc25d8d51765fa74c0da5792dcef481e0f7b5c79 Mon Sep 17 00:00:00 2001 From: naveenaechan Date: Wed, 20 Sep 2023 22:24:13 +0530 Subject: [PATCH 2/8] CLD-5728: Removed netty deps --- common/pom.xml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/common/pom.xml b/common/pom.xml index c5ac435d65..47c274ca4d 100644 --- a/common/pom.xml +++ b/common/pom.xml @@ -138,6 +138,12 @@ org.redisson redisson ${redis.client.version} + + + io.netty + * + + From 67fd1ff3e07d97921cc32b51eedf5e343f904a43 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Thu, 21 Sep 2023 19:58:39 +0530 Subject: [PATCH 3/8] DG-118 Bootstrap inverse policy for Connection link-assets permission --- .../connection_bootstrap_policies.json | 36 +++++++++++++++++++ .../policy_cache_transformer_persona.json | 21 +++++++++++ 2 files changed, 57 insertions(+) diff --git a/addons/static/templates/connection_bootstrap_policies.json b/addons/static/templates/connection_bootstrap_policies.json index fea01add07..4f0b8a448c 100644 --- a/addons/static/templates/connection_bootstrap_policies.json +++ b/addons/static/templates/connection_bootstrap_policies.json @@ -36,6 +36,42 @@ } }, + { + "typeName": "AuthPolicy", + "guid": -8, + "attributes": { + "qualifiedName": "{guid}/connection-link-assets-inverse", + "name": "{name}-connection-link-assets-inverse", + "policyCategory": "bootstrap", + "policySubCategory": "connection", + "policyType": "allow", + "policyServiceName": "atlas", + "policyRoles": [ + "connection_admins_{guid}" + ], + "policyActions": [ + "add-relationship", + "remove-relationship" + ], + "policyResourceCategory": "RELATIONSHIP", + "policyResources": [ + "end-one-entity-classification:*", + "end-one-entity:*", + "end-one-entity-type:Catalog", + "end-one-entity-type:Connection", + "end-one-entity-type:Process", + "end-one-entity-type:ProcessExecution", + "end-one-entity-type:Namespace", + + "end-two-entity-classification:*", + "end-two-entity:{entity}", + "end-two-entity:{entity}/*", + "end-two-entity-type:*", + + "relationship-type:*" + ] + } + }, { "typeName": "AuthPolicy", "guid": -2, diff --git a/addons/static/templates/policy_cache_transformer_persona.json b/addons/static/templates/policy_cache_transformer_persona.json index 926fab339a..e48eba784a 100644 --- a/addons/static/templates/policy_cache_transformer_persona.json +++ b/addons/static/templates/policy_cache_transformer_persona.json @@ -44,6 +44,27 @@ "end-two-entity:*" ], "actions": ["add-relationship", "remove-relationship"] + }, + { + "policyType": "ACCESS", + "policyResourceCategory": "RELATIONSHIP", + "resources": [ + "relationship-type:*", + + "end-one-entity-type:Catalog", + "end-one-entity-type:Connection", + "end-one-entity-type:Process", + "end-one-entity-type:Namespace", + "end-one-entity-type:ProcessExecution", + "end-one-entity-classification:*", + "end-one-entity:*", + + "end-two-entity-type:{entity-type}", + "end-two-entity-classification:*", + "end-two-entity:{entity}", + "end-two-entity:{entity}/*" + ], + "actions": ["add-relationship", "remove-relationship"] } ], "persona-api-create": [ From 830cde45044b5e9e5cb08d7b05930d6405b31c70 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Mon, 25 Sep 2023 20:51:52 +0530 Subject: [PATCH 4/8] DG-128 Fix __hasLineage filter issue --- .../repository/store/graph/v1/DeleteHandlerV1.java | 4 +++- .../main/java/org/apache/atlas/RequestContext.java | 11 +++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java index 0a6cbccd2a..d4437922b6 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v1/DeleteHandlerV1.java @@ -1527,6 +1527,8 @@ public void resetHasLineageOnInputOutputDelete(Collection removedEdge private void updateAssetHasLineageStatus(AtlasVertex assetVertex, AtlasEdge currentEdge, Collection removedEdges) { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("updateAssetHasLineageStatus"); + removedEdges.forEach(edge -> RequestContext.get().addToDeletedEdgesIdsForResetHasLineage(edge.getIdForDisplay())); + Iterator edgeIterator = assetVertex.query() .direction(AtlasEdgeDirection.BOTH) .label(PROCESS_EDGE_LABELS) @@ -1537,7 +1539,7 @@ private void updateAssetHasLineageStatus(AtlasVertex assetVertex, AtlasEdge curr int processHasLineageCount = 0; while (edgeIterator.hasNext()) { AtlasEdge edge = edgeIterator.next(); - if (!removedEdges.contains(edge) && !currentEdge.equals(edge)) { + if (!RequestContext.get().getDeletedEdgesIdsForResetHasLineage().contains(edge.getIdForDisplay()) && !currentEdge.equals(edge)) { AtlasVertex relatedProcessVertex = edge.getOutVertex(); boolean processHasLineage = getEntityHasLineage(relatedProcessVertex); if (processHasLineage) { diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java b/server-api/src/main/java/org/apache/atlas/RequestContext.java index e3cbeb5d94..99568e89ee 100644 --- a/server-api/src/main/java/org/apache/atlas/RequestContext.java +++ b/server-api/src/main/java/org/apache/atlas/RequestContext.java @@ -94,6 +94,8 @@ public class RequestContext { private boolean allowDuplicateDisplayName; private MetricsRegistry metricsRegistry; private boolean skipAuthorizationCheck = false; + private Set deletedEdgesIdsForResetHasLineage = new HashSet<>(0); + private RequestContext() { } @@ -147,6 +149,7 @@ public void clearCache() { this.removedElementsMap.clear(); this.deletedEdgesIds.clear(); this.processGuidIds.clear(); + this.deletedEdgesIdsForResetHasLineage.clear(); this.requestContextHeaders.clear(); this.relationshipEndToVertexIdMap.clear(); this.relationshipMutationMap.clear(); @@ -389,6 +392,14 @@ public Set getDeletedEdgesIds() { return deletedEdgesIds; } + public void addToDeletedEdgesIdsForResetHasLineage(String edgeId) { + deletedEdgesIdsForResetHasLineage.add(edgeId); + } + + public Set getDeletedEdgesIdsForResetHasLineage() { + return deletedEdgesIdsForResetHasLineage; + } + public Set getProcessGuidIds() { return processGuidIds; } From d5960c805764079b71c8d82610596d75b3b95586 Mon Sep 17 00:00:00 2001 From: suraj5077 Date: Thu, 28 Sep 2023 15:51:42 +0530 Subject: [PATCH 5/8] fix: skip BFS iteration if vertex contains null guid --- .../apache/atlas/discovery/EntityLineageService.java | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index a1fa489129..420e4807cd 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -494,12 +494,16 @@ private void enqueueNeighbours(AtlasVertex currentVertex, boolean isDataset, Atl else neighbourVertex = currentEdge.getInVertex(); + String vertexGuid = getGuid(neighbourVertex); + if (StringUtils.isEmpty(vertexGuid)) + continue; + if (!lineageListContext.evaluateTraversalFilter(neighbourVertex)) continue; - if (!skippedVertices.contains(getGuid(neighbourVertex)) && !visitedVertices.contains(getGuid(neighbourVertex))) { - visitedVertices.add(getGuid(neighbourVertex)); - traversalQueue.add(getGuid(neighbourVertex)); + if (!skippedVertices.contains(vertexGuid) && !visitedVertices.contains(vertexGuid)) { + visitedVertices.add(vertexGuid); + traversalQueue.add(vertexGuid); addEntitiesToCache(neighbourVertex); } } From 5d187cea3c6c61da45f98ebe6bc9eda18cbebb9a Mon Sep 17 00:00:00 2001 From: suraj5077 Date: Thu, 28 Sep 2023 16:04:50 +0530 Subject: [PATCH 6/8] fix: refactoring null check for vertices with null guids --- .../org/apache/atlas/discovery/EntityLineageService.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index 420e4807cd..084a8908e1 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -495,10 +495,7 @@ private void enqueueNeighbours(AtlasVertex currentVertex, boolean isDataset, Atl neighbourVertex = currentEdge.getInVertex(); String vertexGuid = getGuid(neighbourVertex); - if (StringUtils.isEmpty(vertexGuid)) - continue; - - if (!lineageListContext.evaluateTraversalFilter(neighbourVertex)) + if (StringUtils.isEmpty(vertexGuid) || !lineageListContext.evaluateTraversalFilter(neighbourVertex)) continue; if (!skippedVertices.contains(vertexGuid) && !visitedVertices.contains(vertexGuid)) { From 89462d8759df67fac5237db2481c4f57742df5fe Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Thu, 28 Sep 2023 20:27:57 +0530 Subject: [PATCH 7/8] DG-128 repair __hasLineage for assets (cherry picked from commit 1b5d31a1730045cb087e3cece61d8c5cafcad0e5) --- .../instance/AtlasHasLineageRequest.java | 10 +++ .../store/graph/v2/AtlasEntityStoreV2.java | 63 ++++++++++++++----- 2 files changed, 58 insertions(+), 15 deletions(-) diff --git a/intg/src/main/java/org/apache/atlas/model/instance/AtlasHasLineageRequest.java b/intg/src/main/java/org/apache/atlas/model/instance/AtlasHasLineageRequest.java index 4340dd8523..42f97dba33 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/AtlasHasLineageRequest.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/AtlasHasLineageRequest.java @@ -40,6 +40,7 @@ public class AtlasHasLineageRequest implements Serializable { public AtlasHasLineageRequest() { } + private String assetGuid; private String processGuid; private String endGuid; private String label; @@ -68,9 +69,18 @@ public void setLabel(String label) { this.label = label; } + public String getAssetGuid() { + return assetGuid; + } + + public void setAssetGuid(String assetGuid) { + this.assetGuid = assetGuid; + } + @Override public String toString() { return "AtlasHasLineageRequest{" + + "assetGuid='" + assetGuid + '\'' + "processGuid='" + processGuid + '\'' + ", endGuid='" + endGuid + '\'' + ", label='" + label + '\'' + diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 88dedd45a9..6c4734082a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -2443,28 +2443,61 @@ public void repairHasLineage(AtlasHasLineageRequests requests) throws AtlasBaseE Set inputOutputEdges = new HashSet<>(); for (AtlasHasLineageRequest request : requests.getRequest()) { - AtlasVertex processVertex = AtlasGraphUtilsV2.findByGuid(this.graph, request.getProcessGuid()); - AtlasVertex assetVertex = AtlasGraphUtilsV2.findByGuid(this.graph, request.getEndGuid()); - AtlasEdge edge = null; - try { - if (processVertex != null && assetVertex != null) { - edge = graphHelper.getEdge(processVertex, assetVertex, request.getLabel()); - } else { - LOG.warn("Skipping since vertex is null for processGuid {} and asset Guid {}" - ,request.getProcessGuid(),request.getEndGuid() ); + if (StringUtils.isNotEmpty(request.getAssetGuid())) { + //only supports repairing scenario mentioned here - https://atlanhq.atlassian.net/browse/DG-128?focusedCommentId=20652 + repairHasLineageForAsset(request); + + } else { + AtlasVertex processVertex = AtlasGraphUtilsV2.findByGuid(this.graph, request.getProcessGuid()); + AtlasVertex assetVertex = AtlasGraphUtilsV2.findByGuid(this.graph, request.getEndGuid()); + AtlasEdge edge = null; + try { + if (processVertex != null && assetVertex != null) { + edge = graphHelper.getEdge(processVertex, assetVertex, request.getLabel()); + } else { + LOG.warn("Skipping since vertex is null for processGuid {} and asset Guid {}" + ,request.getProcessGuid(),request.getEndGuid() ); + } + } catch (RepositoryException re) { + throw new AtlasBaseException(AtlasErrorCode.HAS_LINEAGE_GET_EDGE_FAILED, re); } - } catch (RepositoryException re) { - throw new AtlasBaseException(AtlasErrorCode.HAS_LINEAGE_GET_EDGE_FAILED, re); - } - if (edge != null) { - inputOutputEdges.add(edge); + if (edge != null) { + inputOutputEdges.add(edge); + } } } - repairHasLineageWithAtlasEdges(inputOutputEdges); + + if (CollectionUtils.isNotEmpty(inputOutputEdges)) { + repairHasLineageWithAtlasEdges(inputOutputEdges); + } + RequestContext.get().endMetricRecord(metricRecorder); } + private void repairHasLineageForAsset(AtlasHasLineageRequest request) { + //only supports repairing scenario mentioned here - https://atlanhq.atlassian.net/browse/DG-128?focusedCommentId=20652 + + AtlasVertex assetVertex = AtlasGraphUtilsV2.findByGuid(this.graph, request.getAssetGuid()); + + if (getEntityHasLineage(assetVertex)) { + Iterator lineageEdges = assetVertex.getEdges(AtlasEdgeDirection.BOTH, PROCESS_EDGE_LABELS).iterator(); + boolean foundActiveRel = false; + + while (lineageEdges.hasNext()) { + AtlasEdge edge = lineageEdges.next(); + if (getStatus(edge) == ACTIVE) { + foundActiveRel = true; + break; + } + } + + if (!foundActiveRel) { + AtlasGraphUtilsV2.setEncodedProperty(assetVertex, HAS_LINEAGE, false); + } + } + } + public void repairHasLineageWithAtlasEdges(Set inputOutputEdges) { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("repairHasLineageWithAtlasEdges"); From b42c5b05a9be44d7fb76dd90df21eab2c88d27b2 Mon Sep 17 00:00:00 2001 From: Nikhil P Bonte Date: Tue, 10 Oct 2023 21:15:20 +0530 Subject: [PATCH 8/8] DG-235 rawQueryText ES indexing issue with \n --- addons/elasticsearch/es-settings.json | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/addons/elasticsearch/es-settings.json b/addons/elasticsearch/es-settings.json index d23de9e945..359482a09d 100644 --- a/addons/elasticsearch/es-settings.json +++ b/addons/elasticsearch/es-settings.json @@ -111,9 +111,10 @@ "replacement":" $0" }, "truncate_filter": { - "pattern": "(^.{0,100000}).*$", + "pattern": "(.{0,100000}).*", "type": "pattern_replace", - "replacement": "$1 " + "replacement": "$1", + "flags": "DOTALL" } } }