From bfdf300a327f9657627e06b6111ad872795ac42e Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Tue, 2 Jul 2024 11:34:00 +0530 Subject: [PATCH 01/26] Add default value to optional fields status and template_version --- .../store/graph/v2/preprocessor/contract/DataContract.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java index 4ceea2853c..bdbac4d268 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java @@ -37,9 +37,9 @@ public class DataContract { @Valid @NotNull public String kind; - public Status status; - @JsonProperty(value = "template_version", defaultValue = "0.0.1") - public String templateVersion; + public Status status = Status.DRAFT; + @JsonProperty(value = "template_version") + public String templateVersion = "0.0.1"; @Valid @NotNull public String data_source; @Valid @NotNull From 8936e5900dff1559ac24fc2e66f7f6c4d431843a Mon Sep 17 00:00:00 2001 From: arpit-at Date: Wed, 3 Jul 2024 20:01:39 +0530 Subject: [PATCH 02/26] DG-1697: Adding endpoint for linking/unlink policy --- .../instance/LinkBusinessPolicyRequest.java | 73 +++++++++++++++++++ .../store/graph/AtlasEntityStore.java | 6 ++ .../store/graph/v2/AtlasEntityStoreV2.java | 21 +++++- .../store/graph/v2/EntityGraphMapper.java | 22 ++++++ .../org/apache/atlas/web/rest/EntityREST.java | 42 +++++++++++ 5 files changed, 162 insertions(+), 2 deletions(-) create mode 100644 intg/src/main/java/org/apache/atlas/model/instance/LinkBusinessPolicyRequest.java diff --git a/intg/src/main/java/org/apache/atlas/model/instance/LinkBusinessPolicyRequest.java b/intg/src/main/java/org/apache/atlas/model/instance/LinkBusinessPolicyRequest.java new file mode 100644 index 0000000000..413e9ec22f --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/model/instance/LinkBusinessPolicyRequest.java @@ -0,0 +1,73 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.model.instance; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.io.Serializable; +import java.util.List; + +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; + +/** + * Request to link/unlink policies from asset. + */ +@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +@XmlRootElement +@XmlAccessorType(XmlAccessType.PROPERTY) +public class LinkBusinessPolicyRequest implements Serializable { + private static final long serialVersionUID = 1L; + + private List linkGuids; + private List unlinkGuids; + + public List getLinkGuids() { + return linkGuids; + } + + public void setLinkGuids(List linkGuids) { + this.linkGuids = linkGuids; + } + + public List getUnlinkGuids() { + return unlinkGuids; + } + + public void setUnlinkGuids(List unlinkGuids) { + this.unlinkGuids = unlinkGuids; + } + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("LinkBusinessPolicyRequest{"); + sb.append("linkGuids=").append(linkGuids); + sb.append(", unlinkGuids=").append(unlinkGuids); + sb.append('}'); + return sb.toString(); + } +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java index 912799cdd6..3eb3a42728 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java @@ -365,4 +365,10 @@ EntityMutationResponse deleteByUniqueAttributes(List objectIds) void repairAccesscontrolAlias(String guid) throws AtlasBaseException; + + void linkBusinessPolicy(String policyId, List linkGuids) throws AtlasBaseException; + + + void unlinkBusinessPolicy(String policyId, List unlinkGuids) throws AtlasBaseException; + } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 83df0fe2ef..a227e5d36c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -1550,8 +1550,8 @@ private EntityMutationResponse createOrUpdate(EntityStream entityStream, boolean // Notify the change listeners - entityChangeNotifier.onEntitiesMutated(ret, RequestContext.get().isImportInProgress()); - atlasRelationshipStore.onRelationshipsMutated(RequestContext.get().getRelationshipMutationMap()); + // entityChangeNotifier.onEntitiesMutated(ret, RequestContext.get().isImportInProgress()); + // atlasRelationshipStore.onRelationshipsMutated(RequestContext.get().getRelationshipMutationMap()); if (LOG.isDebugEnabled()) { LOG.debug("<== createOrUpdate()"); } @@ -2737,6 +2737,23 @@ public void repairAccesscontrolAlias(String guid) throws AtlasBaseException { RequestContext.get().endMetricRecord(metric); } + + @Override + @GraphTransaction + public void linkBusinessPolicy(String guid, List linkGuids) { + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("linkBusinessPolicy"); + this.entityGraphMapper.linkBusinessPolicy(guid, linkGuids); + RequestContext.get().endMetricRecord(metric); + } + + + @Override + @GraphTransaction + public void unlinkBusinessPolicy(String guid, List unlinkGuids) { + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("linkBusinessPolicy"); + this.entityGraphMapper.unlinkBusinessPolicy(guid, unlinkGuids); + RequestContext.get().endMetricRecord(metric); + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 97ff277872..70835b74cb 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -4514,5 +4514,27 @@ public void addHasLineage(Set inputOutputEdges, boolean isRestoreEnti } RequestContext.get().endMetricRecord(metricRecorder); } + public void linkBusinessPolicy(String policyId, List linkGuids) { + for (String guid : linkGuids) { + AtlasVertex ev = AtlasGraphUtilsV2.findByGuid(graph, guid); + if (ev != null) { + Set existingValues = ev.getMultiValuedSetProperty("assetPolicyGUIDs", String.class); + ev.setProperty("assetPolicyGUIDs", policyId); + ev.setProperty("assetPoliciesCount", existingValues.size() + 1); + updateModificationMetadata(ev); + } + } + } + public void unlinkBusinessPolicy(String policyId, List unlinkGuids) { + for (String guid : unlinkGuids) { + AtlasVertex ev = AtlasGraphUtilsV2.findByGuid(graph, guid); + if (ev != null) { + Set existingValues = ev.getMultiValuedSetProperty("assetPolicyGUIDs", String.class); + ev.removePropertyValue("assetPolicyGUIDs", policyId); + ev.setProperty("assetPoliciesCount", existingValues.size() - 1); + updateModificationMetadata(ev); + } + } + } } \ No newline at end of file diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java index 3aadc06ad2..62c77b142d 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java @@ -75,7 +75,9 @@ import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; import static org.apache.atlas.AtlasErrorCode.DEPRECATED_API; +import static org.apache.atlas.authorize.AtlasAuthorizationUtils.getCurrentUserName; import static org.apache.atlas.authorize.AtlasPrivilege.*; +import static org.apache.atlas.repository.util.AccessControlUtils.ARGO_SERVICE_USER_NAME; /** @@ -1952,7 +1954,47 @@ public void repairAccessControlAlias(@PathParam("guid") String guid) throws Atla } finally { AtlasPerfTracer.log(perf); } + } + + + @POST + @Path("/{policyId}/link-business-policy") + @Produces(Servlets.JSON_MEDIA_TYPE) + @Consumes(Servlets.JSON_MEDIA_TYPE) + @Timed + public void linkBusinessPolicy(@PathParam("policyId") final String policyId, final LinkBusinessPolicyRequest request) throws AtlasBaseException { + if (ARGO_SERVICE_USER_NAME.equals(RequestContext.getCurrentUser())) { + throw new AtlasBaseException(AtlasErrorCode.UNAUTHORIZED_ACCESS, RequestContext.getCurrentUser(), "Policy linking"); + } + AtlasPerfTracer perf = null; + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityREST.linkBusinessPolicy(" + policyId + ")"); + } + entitiesStore.linkBusinessPolicy(policyId, request.getLinkGuids()); + } finally { + AtlasPerfTracer.log(perf); + } + } + @POST + @Path("/{policyId}/unlink-business-policy") + @Produces(Servlets.JSON_MEDIA_TYPE) + @Consumes(Servlets.JSON_MEDIA_TYPE) + @Timed + public void unlinkBusinessPolicy(@PathParam("policyId") final String policyId, final LinkBusinessPolicyRequest request) throws AtlasBaseException { + if (ARGO_SERVICE_USER_NAME.equals(RequestContext.getCurrentUser())) { + throw new AtlasBaseException(AtlasErrorCode.UNAUTHORIZED_ACCESS, RequestContext.getCurrentUser(), "Policy unlinking"); + } + AtlasPerfTracer perf = null; + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityREST.unlinkBusinessPolicy(" + policyId + ")"); + } + entitiesStore.unlinkBusinessPolicy(policyId, request.getUnlinkGuids()); + } finally { + AtlasPerfTracer.log(perf); + } } } From bc5e9339fbf8e0de77d9b301cb0a4dd46bbb0dcd Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Fri, 5 Jul 2024 16:17:16 +0530 Subject: [PATCH 03/26] fix: Ignore classification option validation when propogate is false If propogate option is false then restrict option are ingored and only checked if propogate if true --- .../atlas/repository/store/graph/v2/EntityGraphMapper.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 596420696d..518b721d0f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -3590,6 +3590,7 @@ public void updateClassifications(EntityMutationContext context, String guid, Li if (CollectionUtils.isEmpty(classifications)) { throw new AtlasBaseException(AtlasErrorCode.INVALID_CLASSIFICATION_PARAMS, "update", guid); } + entityRetriever.verifyClassificationsPropagationMode(classifications); AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(this.graph, guid); @@ -3711,7 +3712,6 @@ public void updateClassifications(EntityMutationContext context, String guid, Li Boolean updatedRestrictPropagationThroughLineage = classification.getRestrictPropagationThroughLineage(); Boolean currentRestrictPropagationThroughHierarchy = currentClassification.getRestrictPropagationThroughHierarchy(); Boolean updatedRestrictPropagationThroughHierarchy = classification.getRestrictPropagationThroughHierarchy(); - String propagationMode = entityRetriever.determinePropagationMode(updatedRestrictPropagationThroughLineage, updatedRestrictPropagationThroughHierarchy); if ((!Objects.equals(updatedRemovePropagations, currentRemovePropagations) || !Objects.equals(currentTagPropagation, updatedTagPropagation) || !Objects.equals(currentRestrictPropagationThroughLineage, updatedRestrictPropagationThroughLineage)) && @@ -3731,6 +3731,7 @@ public void updateClassifications(EntityMutationContext context, String guid, Li // compute propagatedEntityVertices once and use it for subsequent iterations and notifications if (updatedTagPropagation != null && (currentTagPropagation != updatedTagPropagation || currentRestrictPropagationThroughLineage != updatedRestrictPropagationThroughLineage || currentRestrictPropagationThroughHierarchy != updatedRestrictPropagationThroughHierarchy)) { if (updatedTagPropagation) { + String propagationMode = entityRetriever.determinePropagationMode(updatedRestrictPropagationThroughLineage, updatedRestrictPropagationThroughHierarchy); if (updatedRestrictPropagationThroughLineage != null && !currentRestrictPropagationThroughLineage && updatedRestrictPropagationThroughLineage) { deleteDelegate.getHandler().removeTagPropagation(classificationVertex); From b255bd711d5cf2e940ce2bf42cb72ad60971e2e1 Mon Sep 17 00:00:00 2001 From: Aayush Sarva Date: Mon, 8 Jul 2024 10:46:37 +0530 Subject: [PATCH 04/26] GRC-25 Remove MEDIUM severity from trivy code scanning alerts (#3308) --- .github/workflows/trivy-docker-scan.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/trivy-docker-scan.yml b/.github/workflows/trivy-docker-scan.yml index 6be78e7552..f910348903 100644 --- a/.github/workflows/trivy-docker-scan.yml +++ b/.github/workflows/trivy-docker-scan.yml @@ -29,7 +29,7 @@ jobs: output: 'trivy-results-docker.sarif' exit-code: '1' #ignore-unfixed: true - severity: 'CRITICAL,HIGH,MEDIUM' + severity: 'CRITICAL,HIGH' - name: Upload Trivy Docker Scan Results To GitHub Security tab uses: github/codeql-action/upload-sarif@v2 From 36e0b18a6b0238508e82f720763ac9f88ea02631 Mon Sep 17 00:00:00 2001 From: Jaivardhan Singh Date: Mon, 8 Jul 2024 14:37:00 +0530 Subject: [PATCH 05/26] chore: add git action for jira id in PR title --- .github/workflows/github-actions-pr-jira.yaml | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .github/workflows/github-actions-pr-jira.yaml diff --git a/.github/workflows/github-actions-pr-jira.yaml b/.github/workflows/github-actions-pr-jira.yaml new file mode 100644 index 0000000000..76cd01ab38 --- /dev/null +++ b/.github/workflows/github-actions-pr-jira.yaml @@ -0,0 +1,14 @@ +name: GitHub-Jira Link Action +run-name: ${{ github.actor }} is ensuring Jira ID is present in PR title +on: + pull_request: + types: [opened, edited, synchronize, reopened] + branches: [main, staging, master, beta, develop, prod, development] + +jobs: + Enforce-GitHub-Jira-Link-Action: + runs-on: ubuntu-latest + if: ${{ !contains(fromJson('["main", "staging", "master", "beta", "develop", "prod", "development"]'), github.event.pull_request.head.ref) }} + steps: + - name: Enforce Pull Request Title includes Jira Issue Key + uses: ryanvade/enforce-pr-title-style-action@v2.1.1 \ No newline at end of file From 8104930a202a62c1375343111a702613397f49d8 Mon Sep 17 00:00:00 2001 From: Bichitra Kumar Sahoo <32828151+bichitra95@users.noreply.github.com> Date: Wed, 10 Jul 2024 14:43:38 +0530 Subject: [PATCH 06/26] DQ-306 Make data_source optional requirement --- .../store/graph/v2/preprocessor/contract/DataContract.java | 1 - 1 file changed, 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java index bdbac4d268..4dce70108a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java @@ -40,7 +40,6 @@ public class DataContract { public Status status = Status.DRAFT; @JsonProperty(value = "template_version") public String templateVersion = "0.0.1"; - @Valid @NotNull public String data_source; @Valid @NotNull public String dataset; From 1cb386248da3f595f78d13d217baf3651b52b2a2 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Wed, 10 Jul 2024 15:56:21 +0530 Subject: [PATCH 07/26] MESH-40 Fixed the TODOs left by previous part of the task. --- .../v2/preprocessor/AuthPolicyPreProcessor.java | 16 ++++++---------- .../datamesh/StakeholderTitlePreProcessor.java | 14 +++++--------- 2 files changed, 11 insertions(+), 19 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java index b8f832d3f2..58fb516564 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java @@ -125,11 +125,9 @@ private void processCreatePolicy(AtlasStruct entity) throws AtlasBaseException { if (!POLICY_SUB_CATEGORY_DOMAIN.equals(policySubCategory)) { validator.validate(policy, null, parentEntity, CREATE); validateConnectionAdmin(policy); + } else { + validateAndReduce(policy); } - // TODO : uncomment after FE release -// else { -// validateAndReduce(policy); -// } policy.setAttribute(QUALIFIED_NAME, String.format("%s/%s", getEntityQualifiedName(parentEntity), getUUID())); @@ -173,11 +171,11 @@ private void validateAndReduce(AtlasEntity policy) { boolean hasAllDomainPattern = resources.stream().anyMatch(resource -> resource.equals("entity:*") || resource.equals("entity:*/super") || - resource.equals("entity:default/domain/*/super") + resource.equals(ENTITY_DEFAULT_DOMAIN_SUPER) ); if (hasAllDomainPattern) { - policy.setAttribute(ATTR_POLICY_RESOURCES, Collections.singletonList("entity:default/domain/*/super")); + policy.setAttribute(ATTR_POLICY_RESOURCES, Collections.singletonList(ENTITY_DEFAULT_DOMAIN_SUPER)); } } @@ -199,11 +197,9 @@ private void processUpdatePolicy(AtlasStruct entity, AtlasVertex vertex) throws if (!POLICY_SUB_CATEGORY_DOMAIN.equals(policySubCategory)) { validator.validate(policy, existingPolicy, parentEntity, UPDATE); validateConnectionAdmin(policy); + } else { + validateAndReduce(policy); } - // TODO : uncomment after FE release -// else { -// validateAndReduce(policy); -// } String qName = getEntityQualifiedName(existingPolicy); policy.setAttribute(QUALIFIED_NAME, qName); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java index 4ece545cb8..1de5fe685e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java @@ -117,15 +117,12 @@ private void processCreateStakeholderTitle(AtlasEntity entity) throws AtlasBaseE } if (domainQualifiedNames.contains(NEW_STAR) || domainQualifiedNames.contains(STAR)) { if (domainQualifiedNames.size() > 1) { - domainQualifiedNames.clear(); - // TODO : convert this to NEW_STAR after FE release - domainQualifiedNames.add(STAR); + domainQualifiedNames.add(NEW_STAR); entity.setAttribute(ATTR_DOMAIN_QUALIFIED_NAMES, domainQualifiedNames); - } // TODO : uncomment this after FE release -// else { -// domainQualifiedNames.replaceAll(s -> s.equals(STAR) ? NEW_STAR : s); -// } + }else { + domainQualifiedNames.replaceAll(s -> s.equals(STAR) ? NEW_STAR : s); + } String qualifiedName = format(PATTERN_QUALIFIED_NAME_ALL_DOMAINS, getUUID()); entity.setAttribute(QUALIFIED_NAME, qualifiedName); @@ -216,8 +213,7 @@ private void authorizeDomainAccess(List domainQualifiedNames) throws Atl String domainQualifiedNameToAuth; if (domainQualifiedNames.contains(STAR) || domainQualifiedNames.contains(NEW_STAR)) { - //TODO : Convert this to NEW_STAR - domainQualifiedNameToAuth = STAR; + domainQualifiedNameToAuth = NEW_STAR; } else { domainQualifiedNameToAuth = domainQualifiedName; } From e31e6b9d3f6c927e552705176de8afbe7161527c Mon Sep 17 00:00:00 2001 From: Nikhil Soni Date: Wed, 10 Jul 2024 14:38:23 +0530 Subject: [PATCH 08/26] Use previous values for missing restrict option If a restrict via lineage or restrict via hierarchy option is missing while updating the set classification, this will use the previously saved value. This is done to avoid bypassing the validation if setting the options one by one. --- .../store/graph/v2/EntityGraphMapper.java | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 518b721d0f..798d86071e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -3712,6 +3712,21 @@ public void updateClassifications(EntityMutationContext context, String guid, Li Boolean updatedRestrictPropagationThroughLineage = classification.getRestrictPropagationThroughLineage(); Boolean currentRestrictPropagationThroughHierarchy = currentClassification.getRestrictPropagationThroughHierarchy(); Boolean updatedRestrictPropagationThroughHierarchy = classification.getRestrictPropagationThroughHierarchy(); + if (updatedRestrictPropagationThroughLineage == null) { + updatedRestrictPropagationThroughLineage = currentRestrictPropagationThroughLineage; + classification.setRestrictPropagationThroughLineage(updatedRestrictPropagationThroughLineage); + } + if (updatedRestrictPropagationThroughHierarchy == null) { + updatedRestrictPropagationThroughHierarchy = currentRestrictPropagationThroughHierarchy; + classification.setRestrictPropagationThroughHierarchy(updatedRestrictPropagationThroughHierarchy); + } + + String propagationMode = CLASSIFICATION_PROPAGATION_MODE_DEFAULT; + if (updatedTagPropagation) { + // determinePropagationMode also validates the propagation restriction option values + propagationMode = entityRetriever.determinePropagationMode(updatedRestrictPropagationThroughLineage, updatedRestrictPropagationThroughHierarchy); + } + if ((!Objects.equals(updatedRemovePropagations, currentRemovePropagations) || !Objects.equals(currentTagPropagation, updatedTagPropagation) || !Objects.equals(currentRestrictPropagationThroughLineage, updatedRestrictPropagationThroughLineage)) && @@ -3731,10 +3746,8 @@ public void updateClassifications(EntityMutationContext context, String guid, Li // compute propagatedEntityVertices once and use it for subsequent iterations and notifications if (updatedTagPropagation != null && (currentTagPropagation != updatedTagPropagation || currentRestrictPropagationThroughLineage != updatedRestrictPropagationThroughLineage || currentRestrictPropagationThroughHierarchy != updatedRestrictPropagationThroughHierarchy)) { if (updatedTagPropagation) { - String propagationMode = entityRetriever.determinePropagationMode(updatedRestrictPropagationThroughLineage, updatedRestrictPropagationThroughHierarchy); if (updatedRestrictPropagationThroughLineage != null && !currentRestrictPropagationThroughLineage && updatedRestrictPropagationThroughLineage) { deleteDelegate.getHandler().removeTagPropagation(classificationVertex); - } if (updatedRestrictPropagationThroughHierarchy != null && !currentRestrictPropagationThroughHierarchy && updatedRestrictPropagationThroughHierarchy) { deleteDelegate.getHandler().removeTagPropagation(classificationVertex); From 5ef2e27f0fc65256642647f526227d9a9be14939 Mon Sep 17 00:00:00 2001 From: Hitesh Khandelwal Date: Wed, 17 Jul 2024 22:46:03 +0530 Subject: [PATCH 09/26] update pat token --- .github/workflows/chart-release-dispatcher.yaml | 10 +++++----- .github/workflows/main-ecr.yml | 2 +- .github/workflows/maven.yml | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/chart-release-dispatcher.yaml b/.github/workflows/chart-release-dispatcher.yaml index 10fc6cbdb0..ecc4348768 100644 --- a/.github/workflows/chart-release-dispatcher.yaml +++ b/.github/workflows/chart-release-dispatcher.yaml @@ -29,7 +29,7 @@ jobs: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v3 with: - token: ${{ secrets.my_pat }} + token: ${{ secrets.ORG_PAT_GITHUB }} ref: ${{ steps.extract_branch.outputs.branch }} fetch-depth: 0 @@ -50,10 +50,10 @@ jobs: - name: Get PR url and PR User id: get_pr_url_user run: | - head_sha=$(curl -s -H "Authorization: Bearer ${{ secrets.my_pat }}" -H "Accept: application/vnd.github.v3+json" "https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }}/jobs" | jq -r '.jobs[0].head_sha') + head_sha=$(curl -s -H "Authorization: Bearer ${{ secrets.ORG_PAT_GITHUB }}" -H "Accept: application/vnd.github.v3+json" "https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }}/jobs" | jq -r '.jobs[0].head_sha') echo "Head SHA: $head_sha" - pr_url=$(curl -s -H "Authorization: Bearer ${{ secrets.my_pat }}" -H "Accept: application/vnd.github.v3+json" "https://api.github.com/search/issues?q=sha:$head_sha+type:pr" | jq -r '.items[0].html_url') - pr_user=$(curl -s -H "Authorization: Bearer ${{ secrets.my_pat }}" -H "Accept: application/vnd.github.v3+json" "https://api.github.com/search/issues?q=sha:$head_sha+type:pr" | jq -r '.items[0].user.login') + pr_url=$(curl -s -H "Authorization: Bearer ${{ secrets.ORG_PAT_GITHUB }}" -H "Accept: application/vnd.github.v3+json" "https://api.github.com/search/issues?q=sha:$head_sha+type:pr" | jq -r '.items[0].html_url') + pr_user=$(curl -s -H "Authorization: Bearer ${{ secrets.ORG_PAT_GITHUB }}" -H "Accept: application/vnd.github.v3+json" "https://api.github.com/search/issues?q=sha:$head_sha+type:pr" | jq -r '.items[0].user.login') echo "pr_url=$pr_url" >> $GITHUB_OUTPUT echo "pr_user=$pr_user" >> $GITHUB_OUTPUT @@ -65,7 +65,7 @@ jobs: - name: Repository Dispatch uses: peter-evans/repository-dispatch@v2 with: - token: ${{ secrets.my_pat }} + token: ${{ secrets.ORG_PAT_GITHUB }} repository: ${{ matrix.repo }} event-type: dispatch_chart_release_workflow client-payload: |- diff --git a/.github/workflows/main-ecr.yml b/.github/workflows/main-ecr.yml index acb8883f8d..2a64a38c27 100644 --- a/.github/workflows/main-ecr.yml +++ b/.github/workflows/main-ecr.yml @@ -196,4 +196,4 @@ jobs: ${{ steps.login-ecr.outputs.registry }}/atlanhq/${{ github.event.repository.name }}:${{ steps.get_branch.outputs.branch }}-${{ steps.semver_tag.outputs.new_tag }} build-args: | ACCESS_TOKEN_USR=$GITHUB_ACTOR - ACCESS_TOKEN_PWD=${{ secrets.my_pat }} \ No newline at end of file + ACCESS_TOKEN_PWD=${{ secrets.ORG_PAT_GITHUB }} \ No newline at end of file diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 0977cb36a2..f8a09b5589 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -58,7 +58,7 @@ jobs: [{ "id": "github", "username": "atlan-ci", - "password": "${{ secrets.my_pat }}" + "password": "${{ secrets.ORG_PAT_GITHUB }}" }] - name: Build with Maven @@ -77,7 +77,7 @@ jobs: shell: bash - name: Get version tag - run: echo "##[set-output name=version;]$(echo `git ls-remote https://${{ secrets.my_pat }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ steps.get_branch.outputs.branch }} | awk '{ print $1}' | cut -c1-7`)abcd" + run: echo "##[set-output name=version;]$(echo `git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ steps.get_branch.outputs.branch }} | awk '{ print $1}' | cut -c1-7`)abcd" id: get_version - name: Set up Buildx @@ -89,7 +89,7 @@ jobs: with: registry: ghcr.io username: $GITHUB_ACTOR - password: ${{ secrets.my_pat }} + password: ${{ secrets.ORG_PAT_GITHUB }} - name: Build and push id: docker_build From 72b591b9794f0ce02f0e4e218f9d9f24ce03405f Mon Sep 17 00:00:00 2001 From: arpit-at Date: Wed, 3 Jul 2024 20:02:07 +0530 Subject: [PATCH 10/26] DG-1697: Added linking and unlinking api for business-policy --- .github/workflows/maven.yml | 1 + .../org/apache/atlas/AtlasConfiguration.java | 1 + .../v2/AlternateLinkingNotifierImpl.java | 67 ++++++++++ .../store/graph/v2/AtlasEntityStoreV2.java | 106 +++++++++------- .../store/graph/v2/EntityGraphMapper.java | 74 +++++++---- .../v2/IAtlasAlternateChangeNotifier.java | 28 +++++ .../graph/v2/bulkimport/MigrationImport.java | 2 +- .../java/org/apache/atlas/RequestContext.java | 86 ++++++++++++- .../EntityNotificationSender.java | 11 +- .../apache/atlas/web/rest/AlternateREST.java | 116 ++++++++++++++++++ .../org/apache/atlas/web/rest/EntityREST.java | 42 ------- 11 files changed, 420 insertions(+), 114 deletions(-) create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AlternateLinkingNotifierImpl.java create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IAtlasAlternateChangeNotifier.java create mode 100644 webapp/src/main/java/org/apache/atlas/web/rest/AlternateREST.java diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 0977cb36a2..1070038c39 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,6 +26,7 @@ on: - development - master - lineageondemand + - policyendpointsmaster jobs: build: diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index a701510a4e..ed54b1bb3f 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -113,6 +113,7 @@ public enum AtlasConfiguration { INDEXSEARCH_ASYNC_SEARCH_KEEP_ALIVE_TIME_IN_SECONDS("atlas.indexsearch.async.search.keep.alive.time.in.seconds", 300), + POLICY_OPERATIONS_NOTIFICATION_MAX_THREADS("atlas.policy.operations.max.threads", 5), ATLAS_MAINTENANCE_MODE("atlas.maintenance.mode", false); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AlternateLinkingNotifierImpl.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AlternateLinkingNotifierImpl.java new file mode 100644 index 0000000000..3ca158b902 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AlternateLinkingNotifierImpl.java @@ -0,0 +1,67 @@ +package org.apache.atlas.repository.store.graph.v2; + +import org.apache.atlas.RequestContext; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.listener.EntityChangeListenerV2; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.model.instance.EntityMutationResponse; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import static org.apache.atlas.repository.graph.GraphHelper.*; + + +@Component +public class AlternateLinkingNotifierImpl implements IAtlasAlternateChangeNotifier { + + private final Set entityChangeListenersV2; + + @Inject + public AlternateLinkingNotifierImpl(Set entityChangeListenersV2) { + this.entityChangeListenersV2 = entityChangeListenersV2; + + } + + @Override + public void onEntitiesMutation(final EntityMutationResponse entityMutationResponse, final Map entityByGuid) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("onEntitiesMutation"); + final List updatedEntities = entityMutationResponse.getUpdatedEntities(); + final List entities = updatedEntities.stream().map(entityHeader -> createAtlasEntity(entityHeader, entityByGuid.get(entityHeader.getGuid()))).collect(Collectors.toList()); + + for (EntityChangeListenerV2 listener : entityChangeListenersV2) { + listener.onEntitiesUpdated(entities, false); + } + + RequestContext.get().endMetricRecord(metricRecorder); + } + + private AtlasEntity createAtlasEntity(AtlasEntityHeader entityHeader, AtlasVertex vertex) { + AtlasEntity atlasEntity = new AtlasEntity(); + atlasEntity.setAttributes(entityHeader.getAttributes()); + atlasEntity.setGuid(entityHeader.getGuid()); + atlasEntity.setTypeName(entityHeader.getTypeName()); + atlasEntity.setStatus(entityHeader.getStatus()); + atlasEntity.setCreatedBy(entityHeader.getCreatedBy()); + atlasEntity.setUpdatedBy(entityHeader.getUpdatedBy()); + atlasEntity.setCreateTime(entityHeader.getCreateTime()); + atlasEntity.setUpdateTime(entityHeader.getUpdateTime()); + atlasEntity.setIsProxy(entityHeader.getIsIncomplete()); + atlasEntity.setIsIncomplete(entityHeader.getIsIncomplete()); + atlasEntity.setProvenanceType(getProvenanceType(vertex)); + atlasEntity.setCustomAttributes(getCustomAttributes(vertex)); + atlasEntity.setHomeId(getHomeId(vertex)); + atlasEntity.setVersion(getVersion(vertex)); + + return atlasEntity; + } + + +} \ No newline at end of file diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index a227e5d36c..b5d8a40d08 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -20,16 +20,12 @@ import com.google.common.annotations.VisibleForTesting; -import org.apache.atlas.AtlasErrorCode; -import org.apache.atlas.DeleteType; -import org.apache.atlas.GraphTransactionInterceptor; -import org.apache.atlas.RequestContext; -import org.apache.atlas.AtlasException; -import org.apache.atlas.AtlasConfiguration; +import org.apache.atlas.*; import org.apache.atlas.annotation.GraphTransaction; import org.apache.atlas.authorize.*; import org.apache.atlas.authorize.AtlasEntityAccessRequest.AtlasEntityAccessRequestBuilder; -import org.apache.atlas.authorize.AtlasPrivilege; +import org.apache.atlas.bulkimport.BulkImportResponse; +import org.apache.atlas.bulkimport.BulkImportResponse.ImportInfo; import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.featureflag.FeatureFlagStore; @@ -55,40 +51,31 @@ import org.apache.atlas.repository.store.graph.EntityGraphDiscovery; import org.apache.atlas.repository.store.graph.EntityGraphDiscoveryContext; import org.apache.atlas.repository.store.graph.v1.DeleteHandlerDelegate; -import org.apache.atlas.repository.store.graph.v2.AtlasEntityComparator.AtlasEntityDiffResult; import org.apache.atlas.repository.store.graph.v1.RestoreHandlerV1; +import org.apache.atlas.repository.store.graph.v2.AtlasEntityComparator.AtlasEntityDiffResult; import org.apache.atlas.repository.store.graph.v2.preprocessor.AuthPolicyPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.ConnectionPreProcessor; -import org.apache.atlas.repository.store.graph.v2.preprocessor.accesscontrol.StakeholderPreProcessor; -import org.apache.atlas.repository.store.graph.v2.preprocessor.contract.ContractPreProcessor; -import org.apache.atlas.repository.store.graph.v2.preprocessor.datamesh.StakeholderTitlePreProcessor; -import org.apache.atlas.repository.store.graph.v2.preprocessor.resource.LinkPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.accesscontrol.PersonaPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.accesscontrol.PurposePreProcessor; -import org.apache.atlas.repository.store.graph.v2.preprocessor.datamesh.DataProductPreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.accesscontrol.StakeholderPreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.contract.ContractPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.datamesh.DataDomainPreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.datamesh.DataProductPreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.datamesh.StakeholderTitlePreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.glossary.CategoryPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.glossary.GlossaryPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.glossary.TermPreProcessor; +import org.apache.atlas.repository.store.graph.v2.preprocessor.resource.LinkPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.resource.ReadmePreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.sql.QueryCollectionPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.sql.QueryFolderPreProcessor; import org.apache.atlas.repository.store.graph.v2.preprocessor.sql.QueryPreProcessor; import org.apache.atlas.repository.store.graph.v2.tasks.MeaningsTask; import org.apache.atlas.tasks.TaskManagement; -import org.apache.atlas.type.AtlasArrayType; -import org.apache.atlas.type.AtlasBusinessMetadataType; +import org.apache.atlas.type.*; import org.apache.atlas.type.AtlasBusinessMetadataType.AtlasBusinessAttribute; -import org.apache.atlas.type.AtlasClassificationType; -import org.apache.atlas.type.AtlasEntityType; -import org.apache.atlas.type.AtlasEnumType; import org.apache.atlas.type.AtlasStructType.AtlasAttribute; -import org.apache.atlas.type.AtlasType; -import org.apache.atlas.type.AtlasTypeRegistry; -import org.apache.atlas.type.AtlasTypeUtil; -import org.apache.atlas.bulkimport.BulkImportResponse; -import org.apache.atlas.bulkimport.BulkImportResponse.ImportInfo; import org.apache.atlas.util.FileUtils; import org.apache.atlas.utils.AtlasEntityUtil; import org.apache.atlas.utils.AtlasPerfMetrics; @@ -111,18 +98,15 @@ import static org.apache.atlas.bulkimport.BulkImportResponse.ImportStatus.FAILED; import static org.apache.atlas.model.instance.AtlasEntity.Status.ACTIVE; import static org.apache.atlas.model.instance.EntityMutations.EntityOperation.*; +import static org.apache.atlas.repository.Constants.IS_INCOMPLETE_PROPERTY_KEY; +import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY; import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.graph.GraphHelper.*; -import static org.apache.atlas.repository.graph.GraphHelper.getStatus; import static org.apache.atlas.repository.store.graph.v2.EntityGraphMapper.validateLabels; -import static org.apache.atlas.repository.store.graph.v2.tasks.MeaningsTaskFactory.*; +import static org.apache.atlas.repository.store.graph.v2.tasks.MeaningsTaskFactory.UPDATE_ENTITY_MEANINGS_ON_TERM_HARD_DELETE; +import static org.apache.atlas.repository.store.graph.v2.tasks.MeaningsTaskFactory.UPDATE_ENTITY_MEANINGS_ON_TERM_SOFT_DELETE; import static org.apache.atlas.repository.util.AccessControlUtils.REL_ATTR_POLICIES; -import static org.apache.atlas.type.Constants.HAS_LINEAGE; -import static org.apache.atlas.type.Constants.HAS_LINEAGE_VALID; -import static org.apache.atlas.type.Constants.MEANINGS_TEXT_PROPERTY_KEY; -import static org.apache.atlas.type.Constants.MEANINGS_PROPERTY_KEY; -import static org.apache.atlas.type.Constants.MEANING_NAMES_PROPERTY_KEY; -import static org.apache.atlas.type.Constants.PENDING_TASKS_PROPERTY_KEY; +import static org.apache.atlas.type.Constants.*; @@ -151,10 +135,12 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore { private final ESAliasStore esAliasStore; + private final IAtlasAlternateChangeNotifier atlasAlternateChangeNotifier; @Inject public AtlasEntityStoreV2(AtlasGraph graph, DeleteHandlerDelegate deleteDelegate, RestoreHandlerV1 restoreHandlerV1, AtlasTypeRegistry typeRegistry, IAtlasEntityChangeNotifier entityChangeNotifier, EntityGraphMapper entityGraphMapper, TaskManagement taskManagement, - AtlasRelationshipStore atlasRelationshipStore, FeatureFlagStore featureFlagStore) { + AtlasRelationshipStore atlasRelationshipStore, FeatureFlagStore featureFlagStore, + IAtlasAlternateChangeNotifier atlasAlternateChangeNotifier) { this.graph = graph; this.deleteDelegate = deleteDelegate; this.restoreHandlerV1 = restoreHandlerV1; @@ -168,7 +154,7 @@ public AtlasEntityStoreV2(AtlasGraph graph, DeleteHandlerDelegate deleteDelegate this.atlasRelationshipStore = atlasRelationshipStore; this.featureFlagStore = featureFlagStore; this.esAliasStore = new ESAliasStore(graph, entityRetriever); - + this.atlasAlternateChangeNotifier = atlasAlternateChangeNotifier; try { this.discovery = new EntityDiscoveryService(typeRegistry, graph, null, null, null, null); } catch (AtlasException e) { @@ -1550,8 +1536,8 @@ private EntityMutationResponse createOrUpdate(EntityStream entityStream, boolean // Notify the change listeners - // entityChangeNotifier.onEntitiesMutated(ret, RequestContext.get().isImportInProgress()); - // atlasRelationshipStore.onRelationshipsMutated(RequestContext.get().getRelationshipMutationMap()); + entityChangeNotifier.onEntitiesMutated(ret, RequestContext.get().isImportInProgress()); + atlasRelationshipStore.onRelationshipsMutated(RequestContext.get().getRelationshipMutationMap()); if (LOG.isDebugEnabled()) { LOG.debug("<== createOrUpdate()"); } @@ -2739,21 +2725,51 @@ public void repairAccesscontrolAlias(String guid) throws AtlasBaseException { } @Override - @GraphTransaction - public void linkBusinessPolicy(String guid, List linkGuids) { - AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("linkBusinessPolicy"); - this.entityGraphMapper.linkBusinessPolicy(guid, linkGuids); - RequestContext.get().endMetricRecord(metric); + public void linkBusinessPolicy(String guid, List linkGuids) throws AtlasBaseException { + processBusinessPolicy(guid, linkGuids, true); } - @Override - @GraphTransaction - public void unlinkBusinessPolicy(String guid, List unlinkGuids) { - AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("linkBusinessPolicy"); - this.entityGraphMapper.unlinkBusinessPolicy(guid, unlinkGuids); + public void unlinkBusinessPolicy(String guid, List unlinkGuids) throws AtlasBaseException { + processBusinessPolicy(guid, unlinkGuids, false); + } + + private void processBusinessPolicy(String guid, List guids, boolean isLink) throws AtlasBaseException { + // Start recording the performance metrics for the operation + String operation = isLink ? "linkBusinessPolicy" : "unlinkBusinessPolicy"; + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord(operation); + + // Link or unlink the business policy to/from entities and retrieve the affected vertices + List vertices = isLink ? this.entityGraphMapper.linkBusinessPolicy(guid, guids) + : this.entityGraphMapper.unlinkBusinessPolicy(guid, guids); + + // If no vertices are returned, exit the method early + if (CollectionUtils.isEmpty(vertices)) { + return; + } + + // Prepare the response for the entity mutations + EntityMutationResponse entityMutationResponse = new EntityMutationResponse(); + for (AtlasVertex vertex : vertices) { + // Convert each vertex to an AtlasEntityHeader and add it to the response + AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeader(vertex); + entityMutationResponse.addEntity(UPDATE, entityHeader); + } + + // Collect the vertices into a map for easier access by their GUID property + Map vertexMap = vertices.stream() + .collect(Collectors.toMap(vertex -> vertex.getProperty("__guid", String.class), vertex -> vertex)); + + // Notify the policy change notifier about the entities that were mutated + this.atlasAlternateChangeNotifier.onEntitiesMutation(entityMutationResponse, vertexMap); + + // End the performance metrics recording RequestContext.get().endMetricRecord(metric); } + + + } + diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 70835b74cb..2ca4519f52 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -98,6 +98,7 @@ import java.util.Set; import java.util.UUID; import java.util.Date; +import java.util.concurrent.CompletableFuture; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -4514,27 +4515,58 @@ public void addHasLineage(Set inputOutputEdges, boolean isRestoreEnti } RequestContext.get().endMetricRecord(metricRecorder); } - public void linkBusinessPolicy(String policyId, List linkGuids) { - for (String guid : linkGuids) { - AtlasVertex ev = AtlasGraphUtilsV2.findByGuid(graph, guid); - if (ev != null) { - Set existingValues = ev.getMultiValuedSetProperty("assetPolicyGUIDs", String.class); - ev.setProperty("assetPolicyGUIDs", policyId); - ev.setProperty("assetPoliciesCount", existingValues.size() + 1); - updateModificationMetadata(ev); - } - } + + + @GraphTransaction + public List linkBusinessPolicy(String policyId, List linkGuids) { + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("linkBusinessPolicy.GraphTransaction"); + List collect = linkGuids.stream().map(guid -> findByGuid(graph, guid)).filter(Objects::nonNull).filter(ev -> { + Set existingValues = ev.getMultiValuedSetProperty("assetPolicyGUIDs", String.class); + return !existingValues.contains(policyId); + }).peek(ev -> { + Set existingValues = ev.getMultiValuedSetProperty("assetPolicyGUIDs", String.class); + existingValues.add(policyId); + ev.setProperty("assetPolicyGUIDs", policyId); + ev.setProperty("assetPoliciesCount", existingValues.size() + 1); + + updateModificationMetadata(ev); + + cacheDifferentialEntity(ev, existingValues, policyId); + }).collect(Collectors.toList()); + RequestContext.get().endMetricRecord(metric); + return collect; } - public void unlinkBusinessPolicy(String policyId, List unlinkGuids) { - for (String guid : unlinkGuids) { - AtlasVertex ev = AtlasGraphUtilsV2.findByGuid(graph, guid); - if (ev != null) { - Set existingValues = ev.getMultiValuedSetProperty("assetPolicyGUIDs", String.class); - ev.removePropertyValue("assetPolicyGUIDs", policyId); - ev.setProperty("assetPoliciesCount", existingValues.size() - 1); - updateModificationMetadata(ev); - } - } + + @GraphTransaction + public List unlinkBusinessPolicy(String policyId, List unlinkGuids) { + + return unlinkGuids.stream().map(guid -> AtlasGraphUtilsV2.findByGuid(graph, guid)).filter(Objects::nonNull).filter(ev -> { + Set existingValues = ev.getMultiValuedSetProperty("assetPolicyGUIDs", String.class); + return existingValues.contains(policyId); + }).peek(ev -> { + Set existingValues = ev.getMultiValuedSetProperty("assetPolicyGUIDs", String.class); + existingValues.remove(policyId); + ev.removePropertyValue("assetPolicyGUIDs", policyId); + ev.setProperty("assetPoliciesCount", existingValues.size() - 1); + + updateModificationMetadata(ev); + + cacheDifferentialEntity(ev, existingValues, policyId); + }).collect(Collectors.toList()); } -} \ No newline at end of file + + + private void cacheDifferentialEntity(AtlasVertex ev, Set existingValues, String policyId) { + AtlasEntity diffEntity = new AtlasEntity(ev.getProperty("__typeName", String.class)); + diffEntity.setGuid(ev.getProperty("__guid", String.class)); + diffEntity.setAttribute("assetPolicyGUIDs", existingValues); + diffEntity.setAttribute("assetPoliciesCount", existingValues.size()); + diffEntity.setUpdatedBy(ev.getProperty(MODIFIED_BY_KEY, String.class)); + diffEntity.setUpdateTime(new Date(ev.getProperty(MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class))); + + RequestContext requestContext = RequestContext.get(); + requestContext.cacheDifferentialEntity(diffEntity); + } + +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IAtlasAlternateChangeNotifier.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IAtlasAlternateChangeNotifier.java new file mode 100644 index 0000000000..3e83666144 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IAtlasAlternateChangeNotifier.java @@ -0,0 +1,28 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.repository.store.graph.v2; + +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.*; +import org.apache.atlas.repository.graphdb.AtlasVertex; + +import java.util.Map; + +public interface IAtlasAlternateChangeNotifier { + void onEntitiesMutation(final EntityMutationResponse entityMutationResponse, final Map entityByGuid) throws AtlasBaseException; +} diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/MigrationImport.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/MigrationImport.java index 0cc7c4a318..cdb485c7bf 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/MigrationImport.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/bulkimport/MigrationImport.java @@ -129,7 +129,7 @@ private AtlasEntityStoreV2 createEntityStore(AtlasGraph graph, AtlasTypeRegistry graph, relationshipStore, entityChangeNotifier, getInstanceConverter(graph), fullTextMapperV2, null, null); AtlasRelationshipStoreV2 atlasRelationshipStoreV2 = new AtlasRelationshipStoreV2(graph, typeRegistry, deleteDelegate, entityChangeNotifier); - return new AtlasEntityStoreV2(graph, deleteDelegate, restoreHandlerV1, typeRegistry, entityChangeNotifier, entityGraphMapper, null, atlasRelationshipStoreV2, null); + return new AtlasEntityStoreV2(graph, deleteDelegate, restoreHandlerV1, typeRegistry, entityChangeNotifier, entityGraphMapper, null, atlasRelationshipStoreV2, null, null); } private void shutdownEntityCreationManager(EntityCreationManager creationManager) { diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java b/server-api/src/main/java/org/apache/atlas/RequestContext.java index 565832b7bd..9ac463ecd1 100644 --- a/server-api/src/main/java/org/apache/atlas/RequestContext.java +++ b/server-api/src/main/java/org/apache/atlas/RequestContext.java @@ -93,7 +93,7 @@ public class RequestContext { private String currentTypePatchAction = ""; private AtlasTask currentTask; private String traceId; - private final Map relationshipEndToVertexIdMap = new HashMap<>(); + private Map relationshipEndToVertexIdMap = new HashMap<>(); private boolean allowDuplicateDisplayName; private MetricsRegistry metricsRegistry; private boolean skipAuthorizationCheck = false; @@ -105,10 +105,82 @@ public class RequestContext { private Map> deletedClassificationAndVertices = new HashMap<>(); private Map> addedClassificationAndVertices = new HashMap<>(); + private boolean isCloned = false; + private boolean alternatePath = false; + private RequestContext() { + + } + + private RequestContext(RequestContext other) { + this.user = other.user; + this.userGroups = other.userGroups != null ? new HashSet<>(other.userGroups) : null; + this.clientIPAddress = other.clientIPAddress; + this.forwardedAddresses = other.forwardedAddresses != null ? new ArrayList<>(other.forwardedAddresses) : null; + this.deleteType = other.deleteType; + this.isPurgeRequested = other.isPurgeRequested; + this.maxAttempts = other.maxAttempts; + this.attemptCount = other.attemptCount; + this.isImportInProgress = other.isImportInProgress; + this.isInNotificationProcessing = other.isInNotificationProcessing; + this.isInTypePatching = other.isInTypePatching; + this.createShellEntityForNonExistingReference = other.createShellEntityForNonExistingReference; + this.skipFailedEntities = other.skipFailedEntities; + this.allowDeletedRelationsIndexsearch = other.allowDeletedRelationsIndexsearch; + this.includeMeanings = other.includeMeanings; + this.includeClassifications = other.includeClassifications; + this.includeClassificationNames = other.includeClassificationNames; + this.currentTypePatchAction = other.currentTypePatchAction; + this.currentTask = other.currentTask; + this.traceId = other.traceId; + this.relationshipEndToVertexIdMap = new HashMap<>(other.relationshipEndToVertexIdMap); + this.allowDuplicateDisplayName = other.allowDuplicateDisplayName; + this.metricsRegistry = other.metricsRegistry; + this.skipAuthorizationCheck = other.skipAuthorizationCheck; + this.deletedEdgesIdsForResetHasLineage = new HashSet<>(other.deletedEdgesIdsForResetHasLineage); + this.requestUri = other.requestUri; + this.cacheEnabled = other.cacheEnabled; + this.delayTagNotifications = other.delayTagNotifications; + this.deletedClassificationAndVertices = new HashMap<>(other.deletedClassificationAndVertices); + this.addedClassificationAndVertices = new HashMap<>(other.addedClassificationAndVertices); + + this.updatedEntities.putAll(other.updatedEntities); + this.deletedEntities.putAll(other.deletedEntities); + this.restoreEntities.putAll(other.restoreEntities); + this.entityCache.putAll(other.entityCache); + this.entityHeaderCache.putAll(other.entityHeaderCache); + this.entityExtInfoCache.putAll(other.entityExtInfoCache); + this.diffEntityCache.putAll(other.diffEntityCache); + this.addedPropagations.putAll(other.addedPropagations); + this.removedPropagations.putAll(other.removedPropagations); + this.requestContextHeaders.putAll(other.requestContextHeaders); + this.deletedEdgesIds.addAll(other.deletedEdgesIds); + this.processGuidIds.addAll(other.processGuidIds); + this.entitiesToSkipUpdate.addAll(other.entitiesToSkipUpdate); + this.onlyCAUpdateEntities.addAll(other.onlyCAUpdateEntities); + this.onlyBAUpdateEntities.addAll(other.onlyBAUpdateEntities); + this.queuedTasks.addAll(other.queuedTasks); + this.relationAttrsForSearch.addAll(other.relationAttrsForSearch); + this.removedElementsMap.putAll(other.removedElementsMap); + this.newElementsCreatedMap.putAll(other.newElementsCreatedMap); + this.relationshipMutationMap.putAll(other.relationshipMutationMap); + } + + + public static RequestContext cloneCurrentContext() { + RequestContext currentContext = get(); + RequestContext requestContext = new RequestContext(currentContext); + requestContext.isCloned = true; + return requestContext; + } + + public static void set(RequestContext context) { + CURRENT_CONTEXT.set(context); } + + //To handle gets from background threads where createContext() is not called //createContext called for every request in the filter public static RequestContext get() { @@ -788,4 +860,16 @@ public void clearMutationContext(String event) { public Map> getRelationshipMutationMap() { return relationshipMutationMap; } + + public boolean isCloned() { + return isCloned; + } + + public boolean isAlternatePath() { + return alternatePath; + } + + public void setAlternatePath(boolean alternatePath) { + this.alternatePath = alternatePath; + } } \ No newline at end of file diff --git a/webapp/src/main/java/org/apache/atlas/notification/EntityNotificationSender.java b/webapp/src/main/java/org/apache/atlas/notification/EntityNotificationSender.java index c039cc837a..7147d23215 100644 --- a/webapp/src/main/java/org/apache/atlas/notification/EntityNotificationSender.java +++ b/webapp/src/main/java/org/apache/atlas/notification/EntityNotificationSender.java @@ -18,6 +18,7 @@ package org.apache.atlas.notification; import org.apache.atlas.GraphTransactionInterceptor; +import org.apache.atlas.RequestContext; import org.apache.atlas.model.notification.EntityNotification; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.configuration.Configuration; @@ -95,10 +96,12 @@ public void send(EntityNotification.EntityNotificationV2.OperationType operation notificationHook = new PostCommitNotificationHook(operationType, notifications); postCommitNotificationHooks.set(notificationHook); } else { - if (isRelationshipEvent(operationType)) - notificationHook.addRelationshipNotifications(notifications); - else - notificationHook.addNotifications(notifications); + if (isRelationshipEvent(operationType)) notificationHook.addRelationshipNotifications(notifications); + else notificationHook.addNotifications(notifications); + } + + if (RequestContext.get().isAlternatePath()) { + notificationHook.onComplete(true); } } diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AlternateREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/AlternateREST.java new file mode 100644 index 0000000000..2746bcbf8c --- /dev/null +++ b/webapp/src/main/java/org/apache/atlas/web/rest/AlternateREST.java @@ -0,0 +1,116 @@ +package org.apache.atlas.web.rest; + +import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.RequestContext; +import org.apache.atlas.annotation.Timed; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.instance.LinkBusinessPolicyRequest; +import org.apache.atlas.repository.store.graph.AtlasEntityStore; +import org.apache.atlas.utils.AtlasPerfTracer; +import org.apache.atlas.web.util.Servlets; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.stereotype.Service; + +import javax.inject.Inject; +import javax.inject.Singleton; +import javax.ws.rs.*; +import javax.ws.rs.core.MediaType; + +import static org.apache.atlas.repository.util.AccessControlUtils.ARGO_SERVICE_USER_NAME; + +@Path("alternate") +@Singleton +@Service +@Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) +@Produces({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) +public class AlternateREST { + + private static final Logger LOG = LoggerFactory.getLogger(AlternateREST.class); + private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("rest.AlternateREST"); + + private final AtlasEntityStore entitiesStore; + + @Inject + public AlternateREST(AtlasEntityStore entitiesStore) { + this.entitiesStore = entitiesStore; + } + + /** + * Links a business policy to entities. + * + * @param policyId the ID of the policy to be linked + * @param request the request containing the GUIDs of the assets to link the policy to + * @throws AtlasBaseException if there is an error during the linking process + */ + @POST + @Path("/{policyId}/link-business-policy") + @Timed + public void linkBusinessPolicy(@PathParam("policyId") final String policyId, final LinkBusinessPolicyRequest request) throws AtlasBaseException { + // Ensure the current user is authorized to link policies + if (!ARGO_SERVICE_USER_NAME.equals(RequestContext.getCurrentUser())) { + throw new AtlasBaseException(AtlasErrorCode.UNAUTHORIZED_ACCESS, RequestContext.getCurrentUser(), "Policy linking"); + } + + // Set request context parameters + RequestContext.get().setAlternatePath(true); + RequestContext.get().setIncludeClassifications(false); + RequestContext.get().setIncludeMeanings(false); + + AtlasPerfTracer perf = null; + try { + // Start performance tracing if enabled + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "AlternateREST.linkBusinessPolicy(" + policyId + ")"); + } + + // Link the business policy to the specified entities + entitiesStore.linkBusinessPolicy(policyId, request.getLinkGuids()); + } catch (AtlasBaseException abe) { + LOG.error("Error in policy linking: ", abe); + throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, "During Policy linking"); + } finally { + // Log performance metrics + AtlasPerfTracer.log(perf); + } + } + + /** + * Unlinks a business policy from entities. + * + * @param policyId the ID of the policy to be unlinked + * @param request the request containing the GUIDs of the assets to unlink the policy from + * @throws AtlasBaseException if there is an error during the unlinking process + */ + @POST + @Path("/{policyId}/unlink-business-policy") + @Timed + public void unlinkBusinessPolicy(@PathParam("policyId") final String policyId, final LinkBusinessPolicyRequest request) throws AtlasBaseException { + // Ensure the current user is authorized to unlink policies + if (!ARGO_SERVICE_USER_NAME.equals(RequestContext.getCurrentUser())) { + throw new AtlasBaseException(AtlasErrorCode.UNAUTHORIZED_ACCESS, RequestContext.getCurrentUser(), "Policy unlinking"); + } + + // Set request context parameters + RequestContext.get().setAlternatePath(true); + RequestContext.get().setIncludeClassifications(false); + RequestContext.get().setIncludeMeanings(false); + + AtlasPerfTracer perf = null; + try { + // Start performance tracing if enabled + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "AlternateREST.unlinkBusinessPolicy(" + policyId + ")"); + } + + // Unlink the business policy from the specified entities + entitiesStore.unlinkBusinessPolicy(policyId, request.getUnlinkGuids()); + } catch (AtlasBaseException abe) { + LOG.error("Error in policy unlinking: ", abe); + throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, "During Policy unlinking"); + } finally { + // Log performance metrics + AtlasPerfTracer.log(perf); + } + } +} diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java index 62c77b142d..7d50b30db7 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java @@ -1955,46 +1955,4 @@ public void repairAccessControlAlias(@PathParam("guid") String guid) throws Atla AtlasPerfTracer.log(perf); } } - - - @POST - @Path("/{policyId}/link-business-policy") - @Produces(Servlets.JSON_MEDIA_TYPE) - @Consumes(Servlets.JSON_MEDIA_TYPE) - @Timed - public void linkBusinessPolicy(@PathParam("policyId") final String policyId, final LinkBusinessPolicyRequest request) throws AtlasBaseException { - if (ARGO_SERVICE_USER_NAME.equals(RequestContext.getCurrentUser())) { - throw new AtlasBaseException(AtlasErrorCode.UNAUTHORIZED_ACCESS, RequestContext.getCurrentUser(), "Policy linking"); - } - AtlasPerfTracer perf = null; - try { - if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityREST.linkBusinessPolicy(" + policyId + ")"); - } - entitiesStore.linkBusinessPolicy(policyId, request.getLinkGuids()); - } finally { - AtlasPerfTracer.log(perf); - } - } - - - @POST - @Path("/{policyId}/unlink-business-policy") - @Produces(Servlets.JSON_MEDIA_TYPE) - @Consumes(Servlets.JSON_MEDIA_TYPE) - @Timed - public void unlinkBusinessPolicy(@PathParam("policyId") final String policyId, final LinkBusinessPolicyRequest request) throws AtlasBaseException { - if (ARGO_SERVICE_USER_NAME.equals(RequestContext.getCurrentUser())) { - throw new AtlasBaseException(AtlasErrorCode.UNAUTHORIZED_ACCESS, RequestContext.getCurrentUser(), "Policy unlinking"); - } - AtlasPerfTracer perf = null; - try { - if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityREST.unlinkBusinessPolicy(" + policyId + ")"); - } - entitiesStore.unlinkBusinessPolicy(policyId, request.getUnlinkGuids()); - } finally { - AtlasPerfTracer.log(perf); - } - } } From 815b407adf9e4f62c6267f371c0032dad1fec765 Mon Sep 17 00:00:00 2001 From: arpit-at Date: Fri, 19 Jul 2024 11:48:33 +0530 Subject: [PATCH 11/26] DG-1697: Added linking and unlinking logic --- intg/src/main/java/org/apache/atlas/AtlasConfiguration.java | 1 - 1 file changed, 1 deletion(-) diff --git a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java index ed54b1bb3f..a701510a4e 100644 --- a/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java +++ b/intg/src/main/java/org/apache/atlas/AtlasConfiguration.java @@ -113,7 +113,6 @@ public enum AtlasConfiguration { INDEXSEARCH_ASYNC_SEARCH_KEEP_ALIVE_TIME_IN_SECONDS("atlas.indexsearch.async.search.keep.alive.time.in.seconds", 300), - POLICY_OPERATIONS_NOTIFICATION_MAX_THREADS("atlas.policy.operations.max.threads", 5), ATLAS_MAINTENANCE_MODE("atlas.maintenance.mode", false); From 5fb0aaf21a39954a7d87b66fa6fc3c0c37f12383 Mon Sep 17 00:00:00 2001 From: arpit-at Date: Fri, 19 Jul 2024 14:18:35 +0530 Subject: [PATCH 12/26] DG-1697: adding for build --- .github/workflows/maven.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 5e07898efa..189de0b9e2 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,7 +26,7 @@ on: - development - master - lineageondemand - - policyendpointsmaster + - policyendpointsmasterss jobs: build: From 89f0deb07ed68fa1adc333775808a999218a377f Mon Sep 17 00:00:00 2001 From: arpit-at Date: Fri, 19 Jul 2024 16:24:48 +0530 Subject: [PATCH 13/26] DG-1697: clean up --- .github/workflows/maven.yml | 1 - .../java/org/apache/atlas/RequestContext.java | 69 +------------------ .../org/apache/atlas/web/rest/EntityREST.java | 2 - 3 files changed, 1 insertion(+), 71 deletions(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 189de0b9e2..f8a09b5589 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,7 +26,6 @@ on: - development - master - lineageondemand - - policyendpointsmasterss jobs: build: diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java b/server-api/src/main/java/org/apache/atlas/RequestContext.java index 9ac463ecd1..62bb0a150f 100644 --- a/server-api/src/main/java/org/apache/atlas/RequestContext.java +++ b/server-api/src/main/java/org/apache/atlas/RequestContext.java @@ -93,7 +93,7 @@ public class RequestContext { private String currentTypePatchAction = ""; private AtlasTask currentTask; private String traceId; - private Map relationshipEndToVertexIdMap = new HashMap<>(); + private final Map relationshipEndToVertexIdMap = new HashMap<>(); private boolean allowDuplicateDisplayName; private MetricsRegistry metricsRegistry; private boolean skipAuthorizationCheck = false; @@ -104,8 +104,6 @@ public class RequestContext { private boolean delayTagNotifications = false; private Map> deletedClassificationAndVertices = new HashMap<>(); private Map> addedClassificationAndVertices = new HashMap<>(); - - private boolean isCloned = false; private boolean alternatePath = false; @@ -113,68 +111,6 @@ private RequestContext() { } - private RequestContext(RequestContext other) { - this.user = other.user; - this.userGroups = other.userGroups != null ? new HashSet<>(other.userGroups) : null; - this.clientIPAddress = other.clientIPAddress; - this.forwardedAddresses = other.forwardedAddresses != null ? new ArrayList<>(other.forwardedAddresses) : null; - this.deleteType = other.deleteType; - this.isPurgeRequested = other.isPurgeRequested; - this.maxAttempts = other.maxAttempts; - this.attemptCount = other.attemptCount; - this.isImportInProgress = other.isImportInProgress; - this.isInNotificationProcessing = other.isInNotificationProcessing; - this.isInTypePatching = other.isInTypePatching; - this.createShellEntityForNonExistingReference = other.createShellEntityForNonExistingReference; - this.skipFailedEntities = other.skipFailedEntities; - this.allowDeletedRelationsIndexsearch = other.allowDeletedRelationsIndexsearch; - this.includeMeanings = other.includeMeanings; - this.includeClassifications = other.includeClassifications; - this.includeClassificationNames = other.includeClassificationNames; - this.currentTypePatchAction = other.currentTypePatchAction; - this.currentTask = other.currentTask; - this.traceId = other.traceId; - this.relationshipEndToVertexIdMap = new HashMap<>(other.relationshipEndToVertexIdMap); - this.allowDuplicateDisplayName = other.allowDuplicateDisplayName; - this.metricsRegistry = other.metricsRegistry; - this.skipAuthorizationCheck = other.skipAuthorizationCheck; - this.deletedEdgesIdsForResetHasLineage = new HashSet<>(other.deletedEdgesIdsForResetHasLineage); - this.requestUri = other.requestUri; - this.cacheEnabled = other.cacheEnabled; - this.delayTagNotifications = other.delayTagNotifications; - this.deletedClassificationAndVertices = new HashMap<>(other.deletedClassificationAndVertices); - this.addedClassificationAndVertices = new HashMap<>(other.addedClassificationAndVertices); - - this.updatedEntities.putAll(other.updatedEntities); - this.deletedEntities.putAll(other.deletedEntities); - this.restoreEntities.putAll(other.restoreEntities); - this.entityCache.putAll(other.entityCache); - this.entityHeaderCache.putAll(other.entityHeaderCache); - this.entityExtInfoCache.putAll(other.entityExtInfoCache); - this.diffEntityCache.putAll(other.diffEntityCache); - this.addedPropagations.putAll(other.addedPropagations); - this.removedPropagations.putAll(other.removedPropagations); - this.requestContextHeaders.putAll(other.requestContextHeaders); - this.deletedEdgesIds.addAll(other.deletedEdgesIds); - this.processGuidIds.addAll(other.processGuidIds); - this.entitiesToSkipUpdate.addAll(other.entitiesToSkipUpdate); - this.onlyCAUpdateEntities.addAll(other.onlyCAUpdateEntities); - this.onlyBAUpdateEntities.addAll(other.onlyBAUpdateEntities); - this.queuedTasks.addAll(other.queuedTasks); - this.relationAttrsForSearch.addAll(other.relationAttrsForSearch); - this.removedElementsMap.putAll(other.removedElementsMap); - this.newElementsCreatedMap.putAll(other.newElementsCreatedMap); - this.relationshipMutationMap.putAll(other.relationshipMutationMap); - } - - - public static RequestContext cloneCurrentContext() { - RequestContext currentContext = get(); - RequestContext requestContext = new RequestContext(currentContext); - requestContext.isCloned = true; - return requestContext; - } - public static void set(RequestContext context) { CURRENT_CONTEXT.set(context); } @@ -861,9 +797,6 @@ public Map> getRelationshipMutationMap() { return relationshipMutationMap; } - public boolean isCloned() { - return isCloned; - } public boolean isAlternatePath() { return alternatePath; diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java index 19038250e6..1fbc18d3a2 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java @@ -75,9 +75,7 @@ import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; import static org.apache.atlas.AtlasErrorCode.DEPRECATED_API; -import static org.apache.atlas.authorize.AtlasAuthorizationUtils.getCurrentUserName; import static org.apache.atlas.authorize.AtlasPrivilege.*; -import static org.apache.atlas.repository.util.AccessControlUtils.ARGO_SERVICE_USER_NAME; /** From 36b8f10152cfbc3836ef545b19aa4644c913dbee Mon Sep 17 00:00:00 2001 From: arpit-at Date: Wed, 24 Jul 2024 14:13:12 +0530 Subject: [PATCH 14/26] DG-1697: PR comments resolved --- .github/workflows/maven.yml | 1 + .../apache/atlas/repository/Constants.java | 4 + .../instance/LinkBusinessPolicyRequest.java | 13 ++-- .../store/graph/AtlasEntityStore.java | 4 +- .../v2/AlternateLinkingNotifierImpl.java | 67 ---------------- .../store/graph/v2/AtlasEntityStoreV2.java | 76 ++++++++++--------- .../graph/v2/BusinessPolicyNotifierImpl.java | 66 ++++++++++++++++ .../store/graph/v2/EntityGraphMapper.java | 45 +++++------ .../v2/IAtlasAlternateChangeNotifier.java | 4 +- .../java/org/apache/atlas/RequestContext.java | 15 ---- .../EntityNotificationSender.java | 4 - ...rnateREST.java => BusinessPolicyREST.java} | 26 +++---- 12 files changed, 156 insertions(+), 169 deletions(-) delete mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AlternateLinkingNotifierImpl.java create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BusinessPolicyNotifierImpl.java rename webapp/src/main/java/org/apache/atlas/web/rest/{AlternateREST.java => BusinessPolicyREST.java} (82%) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index f8a09b5589..189de0b9e2 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,6 +26,7 @@ on: - development - master - lineageondemand + - policyendpointsmasterss jobs: build: diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index accea8ec88..a5529cf397 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -366,6 +366,10 @@ public final class Constants { public static final String IMPALA_SOURCE = "impala"; public static final String STORM_SOURCE = "storm"; public static final String FILE_SPOOL_SOURCE = "file_spool"; + public static final String ASSET_POLICY_GUIDS = "assetPolicyGUIDs"; + public static final String ASSET_POLICIES_COUNT = "assetPoliciesCount"; + + /* * All supported file-format extensions for Bulk Imports through file upload diff --git a/intg/src/main/java/org/apache/atlas/model/instance/LinkBusinessPolicyRequest.java b/intg/src/main/java/org/apache/atlas/model/instance/LinkBusinessPolicyRequest.java index 413e9ec22f..e42fd4ad55 100644 --- a/intg/src/main/java/org/apache/atlas/model/instance/LinkBusinessPolicyRequest.java +++ b/intg/src/main/java/org/apache/atlas/model/instance/LinkBusinessPolicyRequest.java @@ -27,6 +27,7 @@ import javax.xml.bind.annotation.XmlRootElement; import java.io.Serializable; import java.util.List; +import java.util.Set; import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; @@ -43,22 +44,22 @@ public class LinkBusinessPolicyRequest implements Serializable { private static final long serialVersionUID = 1L; - private List linkGuids; - private List unlinkGuids; + private Set linkGuids; + private Set unlinkGuids; - public List getLinkGuids() { + public Set getLinkGuids() { return linkGuids; } - public void setLinkGuids(List linkGuids) { + public void setLinkGuids(Set linkGuids) { this.linkGuids = linkGuids; } - public List getUnlinkGuids() { + public Set getUnlinkGuids() { return unlinkGuids; } - public void setUnlinkGuids(List unlinkGuids) { + public void setUnlinkGuids(Set unlinkGuids) { this.unlinkGuids = unlinkGuids; } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java index 3eb3a42728..24ca97d731 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java @@ -366,9 +366,9 @@ EntityMutationResponse deleteByUniqueAttributes(List objectIds) void repairAccesscontrolAlias(String guid) throws AtlasBaseException; - void linkBusinessPolicy(String policyId, List linkGuids) throws AtlasBaseException; + void linkBusinessPolicy(String policyId, Set linkGuids) throws AtlasBaseException; - void unlinkBusinessPolicy(String policyId, List unlinkGuids) throws AtlasBaseException; + void unlinkBusinessPolicy(String policyId, Set unlinkGuids) throws AtlasBaseException; } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AlternateLinkingNotifierImpl.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AlternateLinkingNotifierImpl.java deleted file mode 100644 index 3ca158b902..0000000000 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AlternateLinkingNotifierImpl.java +++ /dev/null @@ -1,67 +0,0 @@ -package org.apache.atlas.repository.store.graph.v2; - -import org.apache.atlas.RequestContext; -import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.listener.EntityChangeListenerV2; -import org.apache.atlas.model.instance.AtlasEntity; -import org.apache.atlas.model.instance.AtlasEntityHeader; -import org.apache.atlas.model.instance.EntityMutationResponse; -import org.apache.atlas.repository.graphdb.AtlasVertex; -import org.apache.atlas.utils.AtlasPerfMetrics; -import org.springframework.stereotype.Component; - -import javax.inject.Inject; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.stream.Collectors; - -import static org.apache.atlas.repository.graph.GraphHelper.*; - - -@Component -public class AlternateLinkingNotifierImpl implements IAtlasAlternateChangeNotifier { - - private final Set entityChangeListenersV2; - - @Inject - public AlternateLinkingNotifierImpl(Set entityChangeListenersV2) { - this.entityChangeListenersV2 = entityChangeListenersV2; - - } - - @Override - public void onEntitiesMutation(final EntityMutationResponse entityMutationResponse, final Map entityByGuid) throws AtlasBaseException { - AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("onEntitiesMutation"); - final List updatedEntities = entityMutationResponse.getUpdatedEntities(); - final List entities = updatedEntities.stream().map(entityHeader -> createAtlasEntity(entityHeader, entityByGuid.get(entityHeader.getGuid()))).collect(Collectors.toList()); - - for (EntityChangeListenerV2 listener : entityChangeListenersV2) { - listener.onEntitiesUpdated(entities, false); - } - - RequestContext.get().endMetricRecord(metricRecorder); - } - - private AtlasEntity createAtlasEntity(AtlasEntityHeader entityHeader, AtlasVertex vertex) { - AtlasEntity atlasEntity = new AtlasEntity(); - atlasEntity.setAttributes(entityHeader.getAttributes()); - atlasEntity.setGuid(entityHeader.getGuid()); - atlasEntity.setTypeName(entityHeader.getTypeName()); - atlasEntity.setStatus(entityHeader.getStatus()); - atlasEntity.setCreatedBy(entityHeader.getCreatedBy()); - atlasEntity.setUpdatedBy(entityHeader.getUpdatedBy()); - atlasEntity.setCreateTime(entityHeader.getCreateTime()); - atlasEntity.setUpdateTime(entityHeader.getUpdateTime()); - atlasEntity.setIsProxy(entityHeader.getIsIncomplete()); - atlasEntity.setIsIncomplete(entityHeader.getIsIncomplete()); - atlasEntity.setProvenanceType(getProvenanceType(vertex)); - atlasEntity.setCustomAttributes(getCustomAttributes(vertex)); - atlasEntity.setHomeId(getHomeId(vertex)); - atlasEntity.setVersion(getVersion(vertex)); - - return atlasEntity; - } - - -} \ No newline at end of file diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index b5d8a40d08..beadf06db7 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -2725,48 +2725,56 @@ public void repairAccesscontrolAlias(String guid) throws AtlasBaseException { } @Override - public void linkBusinessPolicy(String guid, List linkGuids) throws AtlasBaseException { - processBusinessPolicy(guid, linkGuids, true); - } + @GraphTransaction + public void linkBusinessPolicy(String policyGuid, Set linkGuids) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("linkBusinessPolicy.GraphTransaction"); - @Override - public void unlinkBusinessPolicy(String guid, List unlinkGuids) throws AtlasBaseException { - processBusinessPolicy(guid, unlinkGuids, false); + try { + List vertices = this.entityGraphMapper.linkBusinessPolicy(policyGuid, linkGuids); + if (CollectionUtils.isEmpty(vertices)) { + return; + } + handleBusinessPolicyMutation(vertices); + } catch (Exception e) { + LOG.error("Error during linkBusinessPolicy for policyGuid: {}", policyGuid, e); + throw new AtlasBaseException("Failed to link business policy", e); + } finally { + RequestContext.get().endMetricRecord(metric); + } } - private void processBusinessPolicy(String guid, List guids, boolean isLink) throws AtlasBaseException { - // Start recording the performance metrics for the operation - String operation = isLink ? "linkBusinessPolicy" : "unlinkBusinessPolicy"; - AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord(operation); - - // Link or unlink the business policy to/from entities and retrieve the affected vertices - List vertices = isLink ? this.entityGraphMapper.linkBusinessPolicy(guid, guids) - : this.entityGraphMapper.unlinkBusinessPolicy(guid, guids); + @Override + @GraphTransaction + public void unlinkBusinessPolicy(String policyGuid, Set unlinkGuids) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("unlinkBusinessPolicy.GraphTransaction"); - // If no vertices are returned, exit the method early - if (CollectionUtils.isEmpty(vertices)) { - return; - } + try { + List vertices = this.entityGraphMapper.unlinkBusinessPolicy(policyGuid, unlinkGuids); - // Prepare the response for the entity mutations - EntityMutationResponse entityMutationResponse = new EntityMutationResponse(); - for (AtlasVertex vertex : vertices) { - // Convert each vertex to an AtlasEntityHeader and add it to the response - AtlasEntityHeader entityHeader = entityRetriever.toAtlasEntityHeader(vertex); - entityMutationResponse.addEntity(UPDATE, entityHeader); + if (CollectionUtils.isEmpty(vertices)) { + return; + } + } catch (Exception e) { + LOG.error("Error during unlinkBusinessPolicy for policyGuid: {}", policyGuid, e); + throw new AtlasBaseException("Failed to unlink business policy", e); + } finally { + RequestContext.get().endMetricRecord(metric); } - - // Collect the vertices into a map for easier access by their GUID property - Map vertexMap = vertices.stream() - .collect(Collectors.toMap(vertex -> vertex.getProperty("__guid", String.class), vertex -> vertex)); - - // Notify the policy change notifier about the entities that were mutated - this.atlasAlternateChangeNotifier.onEntitiesMutation(entityMutationResponse, vertexMap); - - // End the performance metrics recording - RequestContext.get().endMetricRecord(metric); } + private void handleBusinessPolicyMutation(List vertices) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("handleBusinessPolicyMutation"); + Map> attributesByGuid = new HashMap<>(0); + vertices.forEach(vertex -> { + String guid = vertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class); + Map attributes = new HashMap<>(0); + attributes.put(ASSET_POLICY_GUIDS, vertex.getMultiValuedSetProperty(ASSET_POLICY_GUIDS, String.class)); + attributes.put(ASSET_POLICIES_COUNT, vertex.getPropertyValues(ASSET_POLICIES_COUNT, Long.class)); + attributesByGuid.put(guid, attributes); + }); + this.atlasAlternateChangeNotifier.onEntitiesMutation(vertices, attributesByGuid); + RequestContext.get().endMetricRecord(metricRecorder); + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BusinessPolicyNotifierImpl.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BusinessPolicyNotifierImpl.java new file mode 100644 index 0000000000..a35cfa2fd2 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BusinessPolicyNotifierImpl.java @@ -0,0 +1,66 @@ +package org.apache.atlas.repository.store.graph.v2; + +import org.apache.atlas.RequestContext; +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.listener.EntityChangeListenerV2; +import org.apache.atlas.model.instance.AtlasEntity; +import org.apache.atlas.model.instance.AtlasEntityHeader; +import org.apache.atlas.model.instance.EntityMutationResponse; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.atlas.utils.AtlasPerfMetrics; +import org.springframework.stereotype.Component; + +import javax.inject.Inject; +import java.util.*; +import java.util.stream.Collectors; + +import static org.apache.atlas.repository.Constants.*; +import static org.apache.atlas.repository.graph.GraphHelper.*; + + +@Component +public class BusinessPolicyNotifierImpl implements IAtlasAlternateChangeNotifier { + + private final Set entityChangeListenersV2; + + @Inject + public BusinessPolicyNotifierImpl(Set entityChangeListenersV2) { + this.entityChangeListenersV2 = entityChangeListenersV2; + + } + + @Override + public void onEntitiesMutation(final List vertices, + final Map> attributesByGuid) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("onEntitiesMutation"); + final List entities = new ArrayList<>(0); + vertices.forEach(item -> entities.add(createAtlasEntity(item, attributesByGuid.get(item.getProperty(GUID_PROPERTY_KEY, String.class))))); + for (EntityChangeListenerV2 listener : entityChangeListenersV2) { + listener.onEntitiesUpdated(entities, false); + } + + RequestContext.get().endMetricRecord(metricRecorder); + } + + private AtlasEntity createAtlasEntity(AtlasVertex vertex, Map attributes) { + AtlasEntity atlasEntity = new AtlasEntity(); + atlasEntity.setAttributes(attributes); + atlasEntity.setGuid(vertex.getProperty(GUID_PROPERTY_KEY, String.class)); + atlasEntity.setTypeName(vertex.getProperty(TYPE_NAME_PROPERTY_KEY, String.class)); + atlasEntity.setStatus(vertex.getProperty(ATTR_CERTIFICATE_STATUS, AtlasEntity.Status.class)); + atlasEntity.setCreatedBy(vertex.getProperty(CREATED_BY_KEY, String.class)); + atlasEntity.setUpdatedBy(vertex.getProperty(MODIFIED_BY_KEY, String.class)); + atlasEntity.setCreateTime(new Date(vertex.getProperty(TIMESTAMP_PROPERTY_KEY, Long.class))); + atlasEntity.setUpdateTime(new Date(vertex.getProperty(MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class))); + atlasEntity.setIsProxy(vertex.getProperty(IS_PROXY_KEY, Boolean.class)); + atlasEntity.setIsIncomplete(vertex.getProperty(IS_INCOMPLETE_PROPERTY_KEY, Boolean.class)); + atlasEntity.setProvenanceType(getProvenanceType(vertex)); + atlasEntity.setCustomAttributes(getCustomAttributes(vertex)); + atlasEntity.setHomeId(getHomeId(vertex)); + atlasEntity.setVersion(getVersion(vertex)); + + return atlasEntity; + } + + +} \ No newline at end of file diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index e7c5bc9740..f52ef2f56d 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -98,7 +98,6 @@ import java.util.Set; import java.util.UUID; import java.util.Date; -import java.util.concurrent.CompletableFuture; import java.util.regex.Matcher; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -4583,53 +4582,47 @@ public void addHasLineage(Set inputOutputEdges, boolean isRestoreEnti } - @GraphTransaction - public List linkBusinessPolicy(String policyId, List linkGuids) { - AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("linkBusinessPolicy.GraphTransaction"); - List collect = linkGuids.stream().map(guid -> findByGuid(graph, guid)).filter(Objects::nonNull).filter(ev -> { - Set existingValues = ev.getMultiValuedSetProperty("assetPolicyGUIDs", String.class); + public List linkBusinessPolicy(String policyId, Set linkGuids) { + return linkGuids.stream().map(guid -> findByGuid(graph, guid)).filter(Objects::nonNull).filter(ev -> { + Set existingValues = ev.getMultiValuedSetProperty(ASSET_POLICY_GUIDS, String.class); return !existingValues.contains(policyId); }).peek(ev -> { - Set existingValues = ev.getMultiValuedSetProperty("assetPolicyGUIDs", String.class); + Set existingValues = ev.getMultiValuedSetProperty(ASSET_POLICY_GUIDS, String.class); existingValues.add(policyId); - ev.setProperty("assetPolicyGUIDs", policyId); - ev.setProperty("assetPoliciesCount", existingValues.size() + 1); + ev.setProperty(ASSET_POLICY_GUIDS, policyId); + ev.setProperty(ASSET_POLICIES_COUNT, existingValues.size()); updateModificationMetadata(ev); - cacheDifferentialEntity(ev, existingValues, policyId); + cacheDifferentialEntity(ev, existingValues); }).collect(Collectors.toList()); - RequestContext.get().endMetricRecord(metric); - return collect; } - @GraphTransaction - public List unlinkBusinessPolicy(String policyId, List unlinkGuids) { - + public List unlinkBusinessPolicy(String policyId, Set unlinkGuids) { return unlinkGuids.stream().map(guid -> AtlasGraphUtilsV2.findByGuid(graph, guid)).filter(Objects::nonNull).filter(ev -> { - Set existingValues = ev.getMultiValuedSetProperty("assetPolicyGUIDs", String.class); + Set existingValues = ev.getMultiValuedSetProperty(ASSET_POLICY_GUIDS, String.class); return existingValues.contains(policyId); }).peek(ev -> { - Set existingValues = ev.getMultiValuedSetProperty("assetPolicyGUIDs", String.class); + Set existingValues = ev.getMultiValuedSetProperty(ASSET_POLICY_GUIDS, String.class); existingValues.remove(policyId); - ev.removePropertyValue("assetPolicyGUIDs", policyId); - ev.setProperty("assetPoliciesCount", existingValues.size() - 1); + ev.removePropertyValue(ASSET_POLICY_GUIDS, policyId); + ev.setProperty(ASSET_POLICIES_COUNT, existingValues.size()); updateModificationMetadata(ev); - cacheDifferentialEntity(ev, existingValues, policyId); + cacheDifferentialEntity(ev, existingValues); }).collect(Collectors.toList()); } - private void cacheDifferentialEntity(AtlasVertex ev, Set existingValues, String policyId) { - AtlasEntity diffEntity = new AtlasEntity(ev.getProperty("__typeName", String.class)); - diffEntity.setGuid(ev.getProperty("__guid", String.class)); - diffEntity.setAttribute("assetPolicyGUIDs", existingValues); - diffEntity.setAttribute("assetPoliciesCount", existingValues.size()); + private void cacheDifferentialEntity(AtlasVertex ev, Set existingValues) { + AtlasEntity diffEntity = new AtlasEntity(ev.getProperty(TYPE_NAME_PROPERTY_KEY, String.class)); + diffEntity.setGuid(ev.getProperty(GUID_PROPERTY_KEY, String.class)); + diffEntity.setAttribute(ASSET_POLICY_GUIDS, existingValues); + diffEntity.setAttribute(ASSET_POLICIES_COUNT, existingValues.size()); diffEntity.setUpdatedBy(ev.getProperty(MODIFIED_BY_KEY, String.class)); - diffEntity.setUpdateTime(new Date(ev.getProperty(MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class))); + diffEntity.setUpdateTime(new Date(RequestContext.get().getRequestTime())); RequestContext requestContext = RequestContext.get(); requestContext.cacheDifferentialEntity(diffEntity); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IAtlasAlternateChangeNotifier.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IAtlasAlternateChangeNotifier.java index 3e83666144..dfca06d3df 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IAtlasAlternateChangeNotifier.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IAtlasAlternateChangeNotifier.java @@ -21,8 +21,10 @@ import org.apache.atlas.model.instance.*; import org.apache.atlas.repository.graphdb.AtlasVertex; +import java.util.List; import java.util.Map; public interface IAtlasAlternateChangeNotifier { - void onEntitiesMutation(final EntityMutationResponse entityMutationResponse, final Map entityByGuid) throws AtlasBaseException; + void onEntitiesMutation(final List vertices, + final Map> attributesByGuid) throws AtlasBaseException; } diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java b/server-api/src/main/java/org/apache/atlas/RequestContext.java index 62bb0a150f..2cb91d72ad 100644 --- a/server-api/src/main/java/org/apache/atlas/RequestContext.java +++ b/server-api/src/main/java/org/apache/atlas/RequestContext.java @@ -104,19 +104,12 @@ public class RequestContext { private boolean delayTagNotifications = false; private Map> deletedClassificationAndVertices = new HashMap<>(); private Map> addedClassificationAndVertices = new HashMap<>(); - private boolean alternatePath = false; private RequestContext() { } - public static void set(RequestContext context) { - CURRENT_CONTEXT.set(context); - } - - - //To handle gets from background threads where createContext() is not called //createContext called for every request in the filter public static RequestContext get() { @@ -797,12 +790,4 @@ public Map> getRelationshipMutationMap() { return relationshipMutationMap; } - - public boolean isAlternatePath() { - return alternatePath; - } - - public void setAlternatePath(boolean alternatePath) { - this.alternatePath = alternatePath; - } } \ No newline at end of file diff --git a/webapp/src/main/java/org/apache/atlas/notification/EntityNotificationSender.java b/webapp/src/main/java/org/apache/atlas/notification/EntityNotificationSender.java index 7147d23215..ece56f294e 100644 --- a/webapp/src/main/java/org/apache/atlas/notification/EntityNotificationSender.java +++ b/webapp/src/main/java/org/apache/atlas/notification/EntityNotificationSender.java @@ -99,10 +99,6 @@ public void send(EntityNotification.EntityNotificationV2.OperationType operation if (isRelationshipEvent(operationType)) notificationHook.addRelationshipNotifications(notifications); else notificationHook.addNotifications(notifications); } - - if (RequestContext.get().isAlternatePath()) { - notificationHook.onComplete(true); - } } class PostCommitNotificationHook extends GraphTransactionInterceptor.PostTransactionHook { diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/AlternateREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/BusinessPolicyREST.java similarity index 82% rename from webapp/src/main/java/org/apache/atlas/web/rest/AlternateREST.java rename to webapp/src/main/java/org/apache/atlas/web/rest/BusinessPolicyREST.java index 2746bcbf8c..82e830be94 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/AlternateREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/BusinessPolicyREST.java @@ -19,41 +19,40 @@ import static org.apache.atlas.repository.util.AccessControlUtils.ARGO_SERVICE_USER_NAME; -@Path("alternate") +@Path("business-policy") @Singleton @Service @Consumes({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) @Produces({Servlets.JSON_MEDIA_TYPE, MediaType.APPLICATION_JSON}) -public class AlternateREST { +public class BusinessPolicyREST { - private static final Logger LOG = LoggerFactory.getLogger(AlternateREST.class); + private static final Logger LOG = LoggerFactory.getLogger(BusinessPolicyREST.class); private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("rest.AlternateREST"); private final AtlasEntityStore entitiesStore; @Inject - public AlternateREST(AtlasEntityStore entitiesStore) { + public BusinessPolicyREST(AtlasEntityStore entitiesStore) { this.entitiesStore = entitiesStore; } /** * Links a business policy to entities. * - * @param policyId the ID of the policy to be linked + * @param policyGuid the ID of the policy to be linked * @param request the request containing the GUIDs of the assets to link the policy to * @throws AtlasBaseException if there is an error during the linking process */ @POST @Path("/{policyId}/link-business-policy") @Timed - public void linkBusinessPolicy(@PathParam("policyId") final String policyId, final LinkBusinessPolicyRequest request) throws AtlasBaseException { + public void linkBusinessPolicy(@PathParam("policyId") final String policyGuid, final LinkBusinessPolicyRequest request) throws AtlasBaseException { // Ensure the current user is authorized to link policies if (!ARGO_SERVICE_USER_NAME.equals(RequestContext.getCurrentUser())) { throw new AtlasBaseException(AtlasErrorCode.UNAUTHORIZED_ACCESS, RequestContext.getCurrentUser(), "Policy linking"); } // Set request context parameters - RequestContext.get().setAlternatePath(true); RequestContext.get().setIncludeClassifications(false); RequestContext.get().setIncludeMeanings(false); @@ -61,11 +60,11 @@ public void linkBusinessPolicy(@PathParam("policyId") final String policyId, fin try { // Start performance tracing if enabled if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "AlternateREST.linkBusinessPolicy(" + policyId + ")"); + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "AlternateREST.linkBusinessPolicy(" + policyGuid + ")"); } // Link the business policy to the specified entities - entitiesStore.linkBusinessPolicy(policyId, request.getLinkGuids()); + entitiesStore.linkBusinessPolicy(policyGuid, request.getLinkGuids()); } catch (AtlasBaseException abe) { LOG.error("Error in policy linking: ", abe); throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, "During Policy linking"); @@ -78,21 +77,20 @@ public void linkBusinessPolicy(@PathParam("policyId") final String policyId, fin /** * Unlinks a business policy from entities. * - * @param policyId the ID of the policy to be unlinked + * @param policyGuid the ID of the policy to be unlinked * @param request the request containing the GUIDs of the assets to unlink the policy from * @throws AtlasBaseException if there is an error during the unlinking process */ @POST @Path("/{policyId}/unlink-business-policy") @Timed - public void unlinkBusinessPolicy(@PathParam("policyId") final String policyId, final LinkBusinessPolicyRequest request) throws AtlasBaseException { + public void unlinkBusinessPolicy(@PathParam("policyId") final String policyGuid, final LinkBusinessPolicyRequest request) throws AtlasBaseException { // Ensure the current user is authorized to unlink policies if (!ARGO_SERVICE_USER_NAME.equals(RequestContext.getCurrentUser())) { throw new AtlasBaseException(AtlasErrorCode.UNAUTHORIZED_ACCESS, RequestContext.getCurrentUser(), "Policy unlinking"); } // Set request context parameters - RequestContext.get().setAlternatePath(true); RequestContext.get().setIncludeClassifications(false); RequestContext.get().setIncludeMeanings(false); @@ -100,11 +98,11 @@ public void unlinkBusinessPolicy(@PathParam("policyId") final String policyId, f try { // Start performance tracing if enabled if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "AlternateREST.unlinkBusinessPolicy(" + policyId + ")"); + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "AlternateREST.unlinkBusinessPolicy(" + policyGuid + ")"); } // Unlink the business policy from the specified entities - entitiesStore.unlinkBusinessPolicy(policyId, request.getUnlinkGuids()); + entitiesStore.unlinkBusinessPolicy(policyGuid, request.getUnlinkGuids()); } catch (AtlasBaseException abe) { LOG.error("Error in policy unlinking: ", abe); throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, "During Policy unlinking"); From 09e7e3dadd31742a988c960210902a8b24ecc66f Mon Sep 17 00:00:00 2001 From: arpit-at Date: Wed, 24 Jul 2024 14:15:45 +0530 Subject: [PATCH 15/26] DG-1697: PR comments resolved --- .../atlas/repository/store/graph/v2/AtlasEntityStoreV2.java | 1 + 1 file changed, 1 insertion(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index beadf06db7..63c0a43b9f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -2754,6 +2754,7 @@ public void unlinkBusinessPolicy(String policyGuid, Set unlinkGuids) thr if (CollectionUtils.isEmpty(vertices)) { return; } + handleBusinessPolicyMutation(vertices); } catch (Exception e) { LOG.error("Error during unlinkBusinessPolicy for policyGuid: {}", policyGuid, e); throw new AtlasBaseException("Failed to unlink business policy", e); From 4a3c887981a13f9c96f19f43ff4b249d58687387 Mon Sep 17 00:00:00 2001 From: arpit-at Date: Wed, 24 Jul 2024 14:19:59 +0530 Subject: [PATCH 16/26] DG-1697: PR comments resolved --- .../atlas/repository/store/graph/v2/AtlasEntityStoreV2.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 63c0a43b9f..73f2f7c526 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -2734,6 +2734,7 @@ public void linkBusinessPolicy(String policyGuid, Set linkGuids) throws if (CollectionUtils.isEmpty(vertices)) { return; } + handleBusinessPolicyMutation(vertices); } catch (Exception e) { LOG.error("Error during linkBusinessPolicy for policyGuid: {}", policyGuid, e); @@ -2754,6 +2755,7 @@ public void unlinkBusinessPolicy(String policyGuid, Set unlinkGuids) thr if (CollectionUtils.isEmpty(vertices)) { return; } + handleBusinessPolicyMutation(vertices); } catch (Exception e) { LOG.error("Error during unlinkBusinessPolicy for policyGuid: {}", policyGuid, e); From 4cb6cc72ab1be5789a3a6cdd756c6be36c62dc70 Mon Sep 17 00:00:00 2001 From: arpit-at Date: Wed, 24 Jul 2024 16:20:23 +0530 Subject: [PATCH 17/26] DG-1697: PR comments resolved --- .../repository/store/graph/v2/AtlasEntityStoreV2.java | 6 ++---- .../store/graph/v2/BusinessPolicyNotifierImpl.java | 2 +- .../org/apache/atlas/web/rest/BusinessPolicyREST.java | 10 ++-------- 3 files changed, 5 insertions(+), 13 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 73f2f7c526..9839e2e369 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -2738,7 +2738,7 @@ public void linkBusinessPolicy(String policyGuid, Set linkGuids) throws handleBusinessPolicyMutation(vertices); } catch (Exception e) { LOG.error("Error during linkBusinessPolicy for policyGuid: {}", policyGuid, e); - throw new AtlasBaseException("Failed to link business policy", e); + throw e; } finally { RequestContext.get().endMetricRecord(metric); } @@ -2748,10 +2748,8 @@ public void linkBusinessPolicy(String policyGuid, Set linkGuids) throws @GraphTransaction public void unlinkBusinessPolicy(String policyGuid, Set unlinkGuids) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("unlinkBusinessPolicy.GraphTransaction"); - try { List vertices = this.entityGraphMapper.unlinkBusinessPolicy(policyGuid, unlinkGuids); - if (CollectionUtils.isEmpty(vertices)) { return; } @@ -2759,7 +2757,7 @@ public void unlinkBusinessPolicy(String policyGuid, Set unlinkGuids) thr handleBusinessPolicyMutation(vertices); } catch (Exception e) { LOG.error("Error during unlinkBusinessPolicy for policyGuid: {}", policyGuid, e); - throw new AtlasBaseException("Failed to unlink business policy", e); + throw e; } finally { RequestContext.get().endMetricRecord(metric); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BusinessPolicyNotifierImpl.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BusinessPolicyNotifierImpl.java index a35cfa2fd2..84ee770d2c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BusinessPolicyNotifierImpl.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BusinessPolicyNotifierImpl.java @@ -47,13 +47,13 @@ private AtlasEntity createAtlasEntity(AtlasVertex vertex, Map at atlasEntity.setAttributes(attributes); atlasEntity.setGuid(vertex.getProperty(GUID_PROPERTY_KEY, String.class)); atlasEntity.setTypeName(vertex.getProperty(TYPE_NAME_PROPERTY_KEY, String.class)); - atlasEntity.setStatus(vertex.getProperty(ATTR_CERTIFICATE_STATUS, AtlasEntity.Status.class)); atlasEntity.setCreatedBy(vertex.getProperty(CREATED_BY_KEY, String.class)); atlasEntity.setUpdatedBy(vertex.getProperty(MODIFIED_BY_KEY, String.class)); atlasEntity.setCreateTime(new Date(vertex.getProperty(TIMESTAMP_PROPERTY_KEY, Long.class))); atlasEntity.setUpdateTime(new Date(vertex.getProperty(MODIFICATION_TIMESTAMP_PROPERTY_KEY, Long.class))); atlasEntity.setIsProxy(vertex.getProperty(IS_PROXY_KEY, Boolean.class)); atlasEntity.setIsIncomplete(vertex.getProperty(IS_INCOMPLETE_PROPERTY_KEY, Boolean.class)); + atlasEntity.setStatus(getStatus(vertex)); atlasEntity.setProvenanceType(getProvenanceType(vertex)); atlasEntity.setCustomAttributes(getCustomAttributes(vertex)); atlasEntity.setHomeId(getHomeId(vertex)); diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/BusinessPolicyREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/BusinessPolicyREST.java index 82e830be94..9c7a94f687 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/BusinessPolicyREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/BusinessPolicyREST.java @@ -40,7 +40,7 @@ public BusinessPolicyREST(AtlasEntityStore entitiesStore) { * Links a business policy to entities. * * @param policyGuid the ID of the policy to be linked - * @param request the request containing the GUIDs of the assets to link the policy to + * @param request the request containing the GUIDs of the assets to link the policy to * @throws AtlasBaseException if there is an error during the linking process */ @POST @@ -65,9 +65,6 @@ public void linkBusinessPolicy(@PathParam("policyId") final String policyGuid, f // Link the business policy to the specified entities entitiesStore.linkBusinessPolicy(policyGuid, request.getLinkGuids()); - } catch (AtlasBaseException abe) { - LOG.error("Error in policy linking: ", abe); - throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, "During Policy linking"); } finally { // Log performance metrics AtlasPerfTracer.log(perf); @@ -78,7 +75,7 @@ public void linkBusinessPolicy(@PathParam("policyId") final String policyGuid, f * Unlinks a business policy from entities. * * @param policyGuid the ID of the policy to be unlinked - * @param request the request containing the GUIDs of the assets to unlink the policy from + * @param request the request containing the GUIDs of the assets to unlink the policy from * @throws AtlasBaseException if there is an error during the unlinking process */ @POST @@ -103,9 +100,6 @@ public void unlinkBusinessPolicy(@PathParam("policyId") final String policyGuid, // Unlink the business policy from the specified entities entitiesStore.unlinkBusinessPolicy(policyGuid, request.getUnlinkGuids()); - } catch (AtlasBaseException abe) { - LOG.error("Error in policy unlinking: ", abe); - throw new AtlasBaseException(AtlasErrorCode.INTERNAL_ERROR, "During Policy unlinking"); } finally { // Log performance metrics AtlasPerfTracer.log(perf); From c9b0274f87ab8a4355602062fe6b7202434ba918 Mon Sep 17 00:00:00 2001 From: arpit-at Date: Wed, 24 Jul 2024 17:22:03 +0530 Subject: [PATCH 18/26] DG-1697: PR comments resolved --- .../store/graph/v2/BusinessPolicyNotifierImpl.java | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BusinessPolicyNotifierImpl.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BusinessPolicyNotifierImpl.java index 84ee770d2c..3b7b096420 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BusinessPolicyNotifierImpl.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BusinessPolicyNotifierImpl.java @@ -34,7 +34,7 @@ public void onEntitiesMutation(final List vertices, final Map> attributesByGuid) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("onEntitiesMutation"); final List entities = new ArrayList<>(0); - vertices.forEach(item -> entities.add(createAtlasEntity(item, attributesByGuid.get(item.getProperty(GUID_PROPERTY_KEY, String.class))))); + vertices.forEach(item -> entities.add(createAtlasEntity(item))); for (EntityChangeListenerV2 listener : entityChangeListenersV2) { listener.onEntitiesUpdated(entities, false); } @@ -42,9 +42,13 @@ public void onEntitiesMutation(final List vertices, RequestContext.get().endMetricRecord(metricRecorder); } - private AtlasEntity createAtlasEntity(AtlasVertex vertex, Map attributes) { + private AtlasEntity createAtlasEntity(AtlasVertex vertex) { AtlasEntity atlasEntity = new AtlasEntity(); + Map attributes = new HashMap<>(); + attributes.put(QUALIFIED_NAME, vertex.getProperty(QUALIFIED_NAME, String.class)); + attributes.put(NAME, vertex.getProperty(NAME, String.class)); atlasEntity.setAttributes(attributes); + atlasEntity.setGuid(vertex.getProperty(GUID_PROPERTY_KEY, String.class)); atlasEntity.setTypeName(vertex.getProperty(TYPE_NAME_PROPERTY_KEY, String.class)); atlasEntity.setCreatedBy(vertex.getProperty(CREATED_BY_KEY, String.class)); @@ -59,6 +63,11 @@ private AtlasEntity createAtlasEntity(AtlasVertex vertex, Map at atlasEntity.setHomeId(getHomeId(vertex)); atlasEntity.setVersion(getVersion(vertex)); + atlasEntity.setAttribute(NAME, vertex.getPropertyValues(NAME, String.class)); + atlasEntity.setAttribute(EntityGraphRetriever.DESCRIPTION, vertex.getPropertyValues(EntityGraphRetriever.DESCRIPTION, String.class)); + atlasEntity.setAttribute(OWNER_ATTRIBUTE, vertex.getPropertyValues(OWNER_ATTRIBUTE, String.class)); + atlasEntity.setAttribute(EntityGraphRetriever.CREATE_TIME, new Date(vertex.getProperty(TIMESTAMP_PROPERTY_KEY, Long.class))); + return atlasEntity; } From bd3fcc88aadf7879758a72da1e9caa77e1370f7b Mon Sep 17 00:00:00 2001 From: arpit-at Date: Wed, 24 Jul 2024 17:37:08 +0530 Subject: [PATCH 19/26] DG-1697: PR comments resolved --- .../repository/store/graph/v2/AtlasEntityStoreV2.java | 5 +---- .../store/graph/v2/BusinessPolicyNotifierImpl.java | 3 +-- .../store/graph/v2/IAtlasAlternateChangeNotifier.java | 9 ++++----- 3 files changed, 6 insertions(+), 11 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 9839e2e369..f9b2f30a7c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -2765,15 +2765,12 @@ public void unlinkBusinessPolicy(String policyGuid, Set unlinkGuids) thr private void handleBusinessPolicyMutation(List vertices) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("handleBusinessPolicyMutation"); - Map> attributesByGuid = new HashMap<>(0); vertices.forEach(vertex -> { - String guid = vertex.getProperty(Constants.GUID_PROPERTY_KEY, String.class); Map attributes = new HashMap<>(0); attributes.put(ASSET_POLICY_GUIDS, vertex.getMultiValuedSetProperty(ASSET_POLICY_GUIDS, String.class)); attributes.put(ASSET_POLICIES_COUNT, vertex.getPropertyValues(ASSET_POLICIES_COUNT, Long.class)); - attributesByGuid.put(guid, attributes); }); - this.atlasAlternateChangeNotifier.onEntitiesMutation(vertices, attributesByGuid); + this.atlasAlternateChangeNotifier.onEntitiesMutation(vertices); RequestContext.get().endMetricRecord(metricRecorder); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BusinessPolicyNotifierImpl.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BusinessPolicyNotifierImpl.java index 3b7b096420..5b55908968 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BusinessPolicyNotifierImpl.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BusinessPolicyNotifierImpl.java @@ -30,8 +30,7 @@ public BusinessPolicyNotifierImpl(Set entityChangeListen } @Override - public void onEntitiesMutation(final List vertices, - final Map> attributesByGuid) throws AtlasBaseException { + public void onEntitiesMutation(final List vertices) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("onEntitiesMutation"); final List entities = new ArrayList<>(0); vertices.forEach(item -> entities.add(createAtlasEntity(item))); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IAtlasAlternateChangeNotifier.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IAtlasAlternateChangeNotifier.java index dfca06d3df..cbaf3a1d6e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IAtlasAlternateChangeNotifier.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IAtlasAlternateChangeNotifier.java @@ -6,9 +6,9 @@ * to you under the Apache License, Version 2.0 (the * "License"); you may not use this file except in compliance * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

* Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. @@ -25,6 +25,5 @@ import java.util.Map; public interface IAtlasAlternateChangeNotifier { - void onEntitiesMutation(final List vertices, - final Map> attributesByGuid) throws AtlasBaseException; + void onEntitiesMutation(final List vertices) throws AtlasBaseException; } From a5e7103bc5ba6f2d731908c6d24f0b1aeef34f23 Mon Sep 17 00:00:00 2001 From: arpit-at Date: Wed, 24 Jul 2024 18:47:12 +0530 Subject: [PATCH 20/26] DG-1697: PR comments resolved --- .../store/graph/v2/BusinessPolicyNotifierImpl.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BusinessPolicyNotifierImpl.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BusinessPolicyNotifierImpl.java index 5b55908968..7de6b94702 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BusinessPolicyNotifierImpl.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BusinessPolicyNotifierImpl.java @@ -43,10 +43,8 @@ public void onEntitiesMutation(final List vertices) throws AtlasBas private AtlasEntity createAtlasEntity(AtlasVertex vertex) { AtlasEntity atlasEntity = new AtlasEntity(); - Map attributes = new HashMap<>(); - attributes.put(QUALIFIED_NAME, vertex.getProperty(QUALIFIED_NAME, String.class)); - attributes.put(NAME, vertex.getProperty(NAME, String.class)); - atlasEntity.setAttributes(attributes); + atlasEntity.setAttribute(QUALIFIED_NAME, vertex.getProperty(QUALIFIED_NAME, String.class)); + atlasEntity.setAttribute(NAME, vertex.getProperty(NAME, String.class)); atlasEntity.setGuid(vertex.getProperty(GUID_PROPERTY_KEY, String.class)); atlasEntity.setTypeName(vertex.getProperty(TYPE_NAME_PROPERTY_KEY, String.class)); @@ -62,9 +60,9 @@ private AtlasEntity createAtlasEntity(AtlasVertex vertex) { atlasEntity.setHomeId(getHomeId(vertex)); atlasEntity.setVersion(getVersion(vertex)); - atlasEntity.setAttribute(NAME, vertex.getPropertyValues(NAME, String.class)); - atlasEntity.setAttribute(EntityGraphRetriever.DESCRIPTION, vertex.getPropertyValues(EntityGraphRetriever.DESCRIPTION, String.class)); - atlasEntity.setAttribute(OWNER_ATTRIBUTE, vertex.getPropertyValues(OWNER_ATTRIBUTE, String.class)); + atlasEntity.setAttribute(NAME, vertex.getProperty(NAME, String.class)); + atlasEntity.setAttribute(EntityGraphRetriever.DESCRIPTION, vertex.getProperty(EntityGraphRetriever.DESCRIPTION, String.class)); + atlasEntity.setAttribute(OWNER_ATTRIBUTE, vertex.getProperty(OWNER_ATTRIBUTE, String.class)); atlasEntity.setAttribute(EntityGraphRetriever.CREATE_TIME, new Date(vertex.getProperty(TIMESTAMP_PROPERTY_KEY, Long.class))); return atlasEntity; From 2869c0b5be5c4a12112e74fe54224943dddc3eac Mon Sep 17 00:00:00 2001 From: arpit-at Date: Wed, 24 Jul 2024 18:59:11 +0530 Subject: [PATCH 21/26] DG-1697: PR comments resolved --- server-api/src/main/java/org/apache/atlas/RequestContext.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java b/server-api/src/main/java/org/apache/atlas/RequestContext.java index 2cb91d72ad..565832b7bd 100644 --- a/server-api/src/main/java/org/apache/atlas/RequestContext.java +++ b/server-api/src/main/java/org/apache/atlas/RequestContext.java @@ -107,7 +107,6 @@ public class RequestContext { private RequestContext() { - } //To handle gets from background threads where createContext() is not called @@ -789,5 +788,4 @@ public void clearMutationContext(String event) { public Map> getRelationshipMutationMap() { return relationshipMutationMap; } - } \ No newline at end of file From 7782517a2bce9b5726de5d88c1cab4944871e4da Mon Sep 17 00:00:00 2001 From: arpit-at Date: Wed, 24 Jul 2024 20:40:21 +0530 Subject: [PATCH 22/26] DG-1697: PR comments resolved --- .../java/org/apache/atlas/web/rest/BusinessPolicyREST.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/BusinessPolicyREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/BusinessPolicyREST.java index 9c7a94f687..8687f07e96 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/BusinessPolicyREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/BusinessPolicyREST.java @@ -6,6 +6,7 @@ import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.LinkBusinessPolicyRequest; import org.apache.atlas.repository.store.graph.AtlasEntityStore; +import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.atlas.utils.AtlasPerfTracer; import org.apache.atlas.web.util.Servlets; import org.slf4j.Logger; @@ -47,6 +48,7 @@ public BusinessPolicyREST(AtlasEntityStore entitiesStore) { @Path("/{policyId}/link-business-policy") @Timed public void linkBusinessPolicy(@PathParam("policyId") final String policyGuid, final LinkBusinessPolicyRequest request) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("linkBusinessPolicy"); // Ensure the current user is authorized to link policies if (!ARGO_SERVICE_USER_NAME.equals(RequestContext.getCurrentUser())) { throw new AtlasBaseException(AtlasErrorCode.UNAUTHORIZED_ACCESS, RequestContext.getCurrentUser(), "Policy linking"); @@ -68,6 +70,7 @@ public void linkBusinessPolicy(@PathParam("policyId") final String policyGuid, f } finally { // Log performance metrics AtlasPerfTracer.log(perf); + RequestContext.get().endMetricRecord(metric); } } @@ -82,6 +85,7 @@ public void linkBusinessPolicy(@PathParam("policyId") final String policyGuid, f @Path("/{policyId}/unlink-business-policy") @Timed public void unlinkBusinessPolicy(@PathParam("policyId") final String policyGuid, final LinkBusinessPolicyRequest request) throws AtlasBaseException { + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("linkBusinessPolicy"); // Ensure the current user is authorized to unlink policies if (!ARGO_SERVICE_USER_NAME.equals(RequestContext.getCurrentUser())) { throw new AtlasBaseException(AtlasErrorCode.UNAUTHORIZED_ACCESS, RequestContext.getCurrentUser(), "Policy unlinking"); @@ -103,6 +107,7 @@ public void unlinkBusinessPolicy(@PathParam("policyId") final String policyGuid, } finally { // Log performance metrics AtlasPerfTracer.log(perf); + RequestContext.get().endMetricRecord(metric); } } } From ca87c6a657f576f4d55d64a6fe62c38662e2bee2 Mon Sep 17 00:00:00 2001 From: arpit-at Date: Wed, 24 Jul 2024 20:40:49 +0530 Subject: [PATCH 23/26] DG-1697: PR comments resolved --- .../main/java/org/apache/atlas/web/rest/BusinessPolicyREST.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/BusinessPolicyREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/BusinessPolicyREST.java index 8687f07e96..883607c8a3 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/BusinessPolicyREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/BusinessPolicyREST.java @@ -85,7 +85,7 @@ public void linkBusinessPolicy(@PathParam("policyId") final String policyGuid, f @Path("/{policyId}/unlink-business-policy") @Timed public void unlinkBusinessPolicy(@PathParam("policyId") final String policyGuid, final LinkBusinessPolicyRequest request) throws AtlasBaseException { - AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("linkBusinessPolicy"); + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("unlinkBusinessPolicy"); // Ensure the current user is authorized to unlink policies if (!ARGO_SERVICE_USER_NAME.equals(RequestContext.getCurrentUser())) { throw new AtlasBaseException(AtlasErrorCode.UNAUTHORIZED_ACCESS, RequestContext.getCurrentUser(), "Policy unlinking"); From e8b73a541129f0a49dc05a78cea00c47d23a8ec8 Mon Sep 17 00:00:00 2001 From: arpit-at Date: Thu, 25 Jul 2024 14:35:15 +0530 Subject: [PATCH 24/26] DG-1697: PR comments resolved --- .github/workflows/maven.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 189de0b9e2..f8a09b5589 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,7 +26,6 @@ on: - development - master - lineageondemand - - policyendpointsmasterss jobs: build: From 2cc48af871fcb9704d70e1c776e3a9094e65ee32 Mon Sep 17 00:00:00 2001 From: arpit-at Date: Thu, 25 Jul 2024 16:28:11 +0530 Subject: [PATCH 25/26] DG-1697: review fix --- .github/workflows/maven.yml | 1 + .../repository/store/graph/v2/AtlasEntityStoreV2.java | 9 ++------- .../store/graph/v2/BusinessPolicyNotifierImpl.java | 10 +--------- ...eNotifier.java => IAtlasMinimalChangeNotifier.java} | 4 +--- .../org/apache/atlas/web/rest/BusinessPolicyREST.java | 6 +++--- 5 files changed, 8 insertions(+), 22 deletions(-) rename repository/src/main/java/org/apache/atlas/repository/store/graph/v2/{IAtlasAlternateChangeNotifier.java => IAtlasMinimalChangeNotifier.java} (90%) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index f8a09b5589..189de0b9e2 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,6 +26,7 @@ on: - development - master - lineageondemand + - policyendpointsmasterss jobs: build: diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index f9b2f30a7c..a5b168f1e7 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -135,12 +135,12 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore { private final ESAliasStore esAliasStore; - private final IAtlasAlternateChangeNotifier atlasAlternateChangeNotifier; + private final IAtlasMinimalChangeNotifier atlasAlternateChangeNotifier; @Inject public AtlasEntityStoreV2(AtlasGraph graph, DeleteHandlerDelegate deleteDelegate, RestoreHandlerV1 restoreHandlerV1, AtlasTypeRegistry typeRegistry, IAtlasEntityChangeNotifier entityChangeNotifier, EntityGraphMapper entityGraphMapper, TaskManagement taskManagement, AtlasRelationshipStore atlasRelationshipStore, FeatureFlagStore featureFlagStore, - IAtlasAlternateChangeNotifier atlasAlternateChangeNotifier) { + IAtlasMinimalChangeNotifier atlasAlternateChangeNotifier) { this.graph = graph; this.deleteDelegate = deleteDelegate; this.restoreHandlerV1 = restoreHandlerV1; @@ -2765,11 +2765,6 @@ public void unlinkBusinessPolicy(String policyGuid, Set unlinkGuids) thr private void handleBusinessPolicyMutation(List vertices) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("handleBusinessPolicyMutation"); - vertices.forEach(vertex -> { - Map attributes = new HashMap<>(0); - attributes.put(ASSET_POLICY_GUIDS, vertex.getMultiValuedSetProperty(ASSET_POLICY_GUIDS, String.class)); - attributes.put(ASSET_POLICIES_COUNT, vertex.getPropertyValues(ASSET_POLICIES_COUNT, Long.class)); - }); this.atlasAlternateChangeNotifier.onEntitiesMutation(vertices); RequestContext.get().endMetricRecord(metricRecorder); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BusinessPolicyNotifierImpl.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BusinessPolicyNotifierImpl.java index 7de6b94702..fab059cfbe 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BusinessPolicyNotifierImpl.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/BusinessPolicyNotifierImpl.java @@ -4,22 +4,19 @@ import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.listener.EntityChangeListenerV2; import org.apache.atlas.model.instance.AtlasEntity; -import org.apache.atlas.model.instance.AtlasEntityHeader; -import org.apache.atlas.model.instance.EntityMutationResponse; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.utils.AtlasPerfMetrics; import org.springframework.stereotype.Component; import javax.inject.Inject; import java.util.*; -import java.util.stream.Collectors; import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.graph.GraphHelper.*; @Component -public class BusinessPolicyNotifierImpl implements IAtlasAlternateChangeNotifier { +public class BusinessPolicyNotifierImpl implements IAtlasMinimalChangeNotifier { private final Set entityChangeListenersV2; @@ -56,14 +53,9 @@ private AtlasEntity createAtlasEntity(AtlasVertex vertex) { atlasEntity.setIsIncomplete(vertex.getProperty(IS_INCOMPLETE_PROPERTY_KEY, Boolean.class)); atlasEntity.setStatus(getStatus(vertex)); atlasEntity.setProvenanceType(getProvenanceType(vertex)); - atlasEntity.setCustomAttributes(getCustomAttributes(vertex)); atlasEntity.setHomeId(getHomeId(vertex)); atlasEntity.setVersion(getVersion(vertex)); - atlasEntity.setAttribute(NAME, vertex.getProperty(NAME, String.class)); - atlasEntity.setAttribute(EntityGraphRetriever.DESCRIPTION, vertex.getProperty(EntityGraphRetriever.DESCRIPTION, String.class)); - atlasEntity.setAttribute(OWNER_ATTRIBUTE, vertex.getProperty(OWNER_ATTRIBUTE, String.class)); - atlasEntity.setAttribute(EntityGraphRetriever.CREATE_TIME, new Date(vertex.getProperty(TIMESTAMP_PROPERTY_KEY, Long.class))); return atlasEntity; } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IAtlasAlternateChangeNotifier.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IAtlasMinimalChangeNotifier.java similarity index 90% rename from repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IAtlasAlternateChangeNotifier.java rename to repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IAtlasMinimalChangeNotifier.java index cbaf3a1d6e..35c2d9e757 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IAtlasAlternateChangeNotifier.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/IAtlasMinimalChangeNotifier.java @@ -18,12 +18,10 @@ package org.apache.atlas.repository.store.graph.v2; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.instance.*; import org.apache.atlas.repository.graphdb.AtlasVertex; import java.util.List; -import java.util.Map; -public interface IAtlasAlternateChangeNotifier { +public interface IAtlasMinimalChangeNotifier { void onEntitiesMutation(final List vertices) throws AtlasBaseException; } diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/BusinessPolicyREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/BusinessPolicyREST.java index 883607c8a3..ef252293a0 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/BusinessPolicyREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/BusinessPolicyREST.java @@ -28,7 +28,7 @@ public class BusinessPolicyREST { private static final Logger LOG = LoggerFactory.getLogger(BusinessPolicyREST.class); - private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("rest.AlternateREST"); + private static final Logger PERF_LOG = AtlasPerfTracer.getPerfLogger("rest.BusinessPolicyREST"); private final AtlasEntityStore entitiesStore; @@ -62,7 +62,7 @@ public void linkBusinessPolicy(@PathParam("policyId") final String policyGuid, f try { // Start performance tracing if enabled if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "AlternateREST.linkBusinessPolicy(" + policyGuid + ")"); + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "BusinessPolicyREST.linkBusinessPolicy(" + policyGuid + ")"); } // Link the business policy to the specified entities @@ -99,7 +99,7 @@ public void unlinkBusinessPolicy(@PathParam("policyId") final String policyGuid, try { // Start performance tracing if enabled if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "AlternateREST.unlinkBusinessPolicy(" + policyGuid + ")"); + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "BusinessPolicyREST.unlinkBusinessPolicy(" + policyGuid + ")"); } // Unlink the business policy from the specified entities From 93e2c1361e0086efa1bf4642299ca25dd55af437 Mon Sep 17 00:00:00 2001 From: arpit-at Date: Thu, 25 Jul 2024 22:28:21 +0530 Subject: [PATCH 26/26] DG-1697: remove unnecesary files --- .github/workflows/maven.yml | 1 - webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java | 2 ++ 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 189de0b9e2..f8a09b5589 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -26,7 +26,6 @@ on: - development - master - lineageondemand - - policyendpointsmasterss jobs: build: diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java index 1fbc18d3a2..c5c95bb98f 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java @@ -1954,5 +1954,7 @@ public void repairAccessControlAlias(@PathParam("guid") String guid) throws Atla } finally { AtlasPerfTracer.log(perf); } + + } }