diff --git a/.github/workflows/chart-release-dispatcher.yaml b/.github/workflows/chart-release-dispatcher.yaml index 10fc6cbdb0..ecc4348768 100644 --- a/.github/workflows/chart-release-dispatcher.yaml +++ b/.github/workflows/chart-release-dispatcher.yaml @@ -29,7 +29,7 @@ jobs: # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it - uses: actions/checkout@v3 with: - token: ${{ secrets.my_pat }} + token: ${{ secrets.ORG_PAT_GITHUB }} ref: ${{ steps.extract_branch.outputs.branch }} fetch-depth: 0 @@ -50,10 +50,10 @@ jobs: - name: Get PR url and PR User id: get_pr_url_user run: | - head_sha=$(curl -s -H "Authorization: Bearer ${{ secrets.my_pat }}" -H "Accept: application/vnd.github.v3+json" "https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }}/jobs" | jq -r '.jobs[0].head_sha') + head_sha=$(curl -s -H "Authorization: Bearer ${{ secrets.ORG_PAT_GITHUB }}" -H "Accept: application/vnd.github.v3+json" "https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }}/jobs" | jq -r '.jobs[0].head_sha') echo "Head SHA: $head_sha" - pr_url=$(curl -s -H "Authorization: Bearer ${{ secrets.my_pat }}" -H "Accept: application/vnd.github.v3+json" "https://api.github.com/search/issues?q=sha:$head_sha+type:pr" | jq -r '.items[0].html_url') - pr_user=$(curl -s -H "Authorization: Bearer ${{ secrets.my_pat }}" -H "Accept: application/vnd.github.v3+json" "https://api.github.com/search/issues?q=sha:$head_sha+type:pr" | jq -r '.items[0].user.login') + pr_url=$(curl -s -H "Authorization: Bearer ${{ secrets.ORG_PAT_GITHUB }}" -H "Accept: application/vnd.github.v3+json" "https://api.github.com/search/issues?q=sha:$head_sha+type:pr" | jq -r '.items[0].html_url') + pr_user=$(curl -s -H "Authorization: Bearer ${{ secrets.ORG_PAT_GITHUB }}" -H "Accept: application/vnd.github.v3+json" "https://api.github.com/search/issues?q=sha:$head_sha+type:pr" | jq -r '.items[0].user.login') echo "pr_url=$pr_url" >> $GITHUB_OUTPUT echo "pr_user=$pr_user" >> $GITHUB_OUTPUT @@ -65,7 +65,7 @@ jobs: - name: Repository Dispatch uses: peter-evans/repository-dispatch@v2 with: - token: ${{ secrets.my_pat }} + token: ${{ secrets.ORG_PAT_GITHUB }} repository: ${{ matrix.repo }} event-type: dispatch_chart_release_workflow client-payload: |- diff --git a/.github/workflows/github-actions-pr-jira.yaml b/.github/workflows/github-actions-pr-jira.yaml new file mode 100644 index 0000000000..76cd01ab38 --- /dev/null +++ b/.github/workflows/github-actions-pr-jira.yaml @@ -0,0 +1,14 @@ +name: GitHub-Jira Link Action +run-name: ${{ github.actor }} is ensuring Jira ID is present in PR title +on: + pull_request: + types: [opened, edited, synchronize, reopened] + branches: [main, staging, master, beta, develop, prod, development] + +jobs: + Enforce-GitHub-Jira-Link-Action: + runs-on: ubuntu-latest + if: ${{ !contains(fromJson('["main", "staging", "master", "beta", "develop", "prod", "development"]'), github.event.pull_request.head.ref) }} + steps: + - name: Enforce Pull Request Title includes Jira Issue Key + uses: ryanvade/enforce-pr-title-style-action@v2.1.1 \ No newline at end of file diff --git a/.github/workflows/main-ecr.yml b/.github/workflows/main-ecr.yml index acb8883f8d..2a64a38c27 100644 --- a/.github/workflows/main-ecr.yml +++ b/.github/workflows/main-ecr.yml @@ -196,4 +196,4 @@ jobs: ${{ steps.login-ecr.outputs.registry }}/atlanhq/${{ github.event.repository.name }}:${{ steps.get_branch.outputs.branch }}-${{ steps.semver_tag.outputs.new_tag }} build-args: | ACCESS_TOKEN_USR=$GITHUB_ACTOR - ACCESS_TOKEN_PWD=${{ secrets.my_pat }} \ No newline at end of file + ACCESS_TOKEN_PWD=${{ secrets.ORG_PAT_GITHUB }} \ No newline at end of file diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index 0977cb36a2..f8a09b5589 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -58,7 +58,7 @@ jobs: [{ "id": "github", "username": "atlan-ci", - "password": "${{ secrets.my_pat }}" + "password": "${{ secrets.ORG_PAT_GITHUB }}" }] - name: Build with Maven @@ -77,7 +77,7 @@ jobs: shell: bash - name: Get version tag - run: echo "##[set-output name=version;]$(echo `git ls-remote https://${{ secrets.my_pat }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ steps.get_branch.outputs.branch }} | awk '{ print $1}' | cut -c1-7`)abcd" + run: echo "##[set-output name=version;]$(echo `git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ steps.get_branch.outputs.branch }} | awk '{ print $1}' | cut -c1-7`)abcd" id: get_version - name: Set up Buildx @@ -89,7 +89,7 @@ jobs: with: registry: ghcr.io username: $GITHUB_ACTOR - password: ${{ secrets.my_pat }} + password: ${{ secrets.ORG_PAT_GITHUB }} - name: Build and push id: docker_build diff --git a/.github/workflows/trivy-docker-scan.yml b/.github/workflows/trivy-docker-scan.yml index 6be78e7552..f910348903 100644 --- a/.github/workflows/trivy-docker-scan.yml +++ b/.github/workflows/trivy-docker-scan.yml @@ -29,7 +29,7 @@ jobs: output: 'trivy-results-docker.sarif' exit-code: '1' #ignore-unfixed: true - severity: 'CRITICAL,HIGH,MEDIUM' + severity: 'CRITICAL,HIGH' - name: Upload Trivy Docker Scan Results To GitHub Security tab uses: github/codeql-action/upload-sarif@v2 diff --git a/common/src/main/java/org/apache/atlas/repository/Constants.java b/common/src/main/java/org/apache/atlas/repository/Constants.java index accea8ec88..4ad2a285f8 100644 --- a/common/src/main/java/org/apache/atlas/repository/Constants.java +++ b/common/src/main/java/org/apache/atlas/repository/Constants.java @@ -159,6 +159,8 @@ public final class Constants { /** * SQL property keys. */ + + public static final String SQL_ENTITY_TYPE = "SQL"; public static final String CONNECTION_ENTITY_TYPE = "Connection"; public static final String QUERY_ENTITY_TYPE = "Query"; public static final String QUERY_FOLDER_ENTITY_TYPE = "Folder"; diff --git a/intg/src/main/java/org/apache/atlas/model/lineage/AtlasLineageOnDemandInfo.java b/intg/src/main/java/org/apache/atlas/model/lineage/AtlasLineageOnDemandInfo.java index 56042a8669..3a86a8a963 100644 --- a/intg/src/main/java/org/apache/atlas/model/lineage/AtlasLineageOnDemandInfo.java +++ b/intg/src/main/java/org/apache/atlas/model/lineage/AtlasLineageOnDemandInfo.java @@ -171,6 +171,8 @@ public static class LineageInfoOnDemand { boolean hasMoreOutputs; int inputRelationsCount; int outputRelationsCount; + int totalInputRelationsCount; + int totalOutputRelationsCount; boolean isInputRelationsReachedLimit; boolean isOutputRelationsReachedLimit; @JsonProperty @@ -188,13 +190,15 @@ public LineageInfoOnDemand(LineageOnDemandConstraints onDemandConstraints) { this.hasMoreOutputs = false; this.inputRelationsCount = 0; this.outputRelationsCount = 0; + this.totalInputRelationsCount = 0; + this.totalOutputRelationsCount = 0; this.isInputRelationsReachedLimit = false; this.isOutputRelationsReachedLimit = false; this.hasUpstream = false; this.hasDownstream = false; this.fromCounter = 0; } - + public boolean isInputRelationsReachedLimit() { return isInputRelationsReachedLimit; } @@ -243,10 +247,18 @@ public void setHasDownstream(boolean hasDownstream) { this.hasDownstream = hasDownstream; } - public int getFromCounter() { - return fromCounter; + public int getTotalInputRelationsCount() { + return totalInputRelationsCount; + } + + public void setTotalInputRelationsCount(int count) {this.totalInputRelationsCount = count;} + + public int getTotalOutputRelationsCount() { + return totalOutputRelationsCount; } + public void setTotalOutputRelationsCount(int count) {this.totalOutputRelationsCount = count;} + public void incrementFromCounter() { fromCounter++; } @@ -255,6 +267,10 @@ public int getInputRelationsCount() { return inputRelationsCount; } + public int getFromCounter() { + return fromCounter; + } + public void incrementInputRelationsCount() { this.inputRelationsCount++; if (inputRelationsCount == onDemandConstraints.getInputRelationsLimit()) { diff --git a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java index de939eac55..9c4b07764c 100644 --- a/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java +++ b/repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java @@ -343,7 +343,7 @@ private void traverseEdgesOnDemand(Iterator processEdges, boolean isI } boolean isInputEdge = processEdge.getLabel().equalsIgnoreCase(PROCESS_INPUTS_EDGE); - if (incrementAndCheckIfRelationsLimitReached(processEdge, isInputEdge, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, direction)) { + if (incrementAndCheckIfRelationsLimitReached(processEdge, isInputEdge, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, direction, new HashSet<>())) { break; } else { addEdgeToResult(processEdge, ret, atlasLineageOnDemandContext, nextLevel, traversalOrder); @@ -387,7 +387,7 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i continue; } - if (incrementAndCheckIfRelationsLimitReached(incomingEdge, !isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, direction)) { + if (incrementAndCheckIfRelationsLimitReached(incomingEdge, !isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, direction, visitedVertices)) { LineageInfoOnDemand entityOnDemandInfo = ret.getRelationsOnDemand().get(baseGuid); if (entityOnDemandInfo == null) continue; @@ -414,7 +414,7 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i if (checkForOffset(outgoingEdge, processVertex, atlasLineageOnDemandContext, ret)) { continue; } - if (incrementAndCheckIfRelationsLimitReached(outgoingEdge, isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, direction)) { + if (incrementAndCheckIfRelationsLimitReached(outgoingEdge, isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, direction, visitedVertices)) { String processGuid = AtlasGraphUtilsV2.getIdFromVertex(processVertex); LineageInfoOnDemand entityOnDemandInfo = ret.getRelationsOnDemand().get(processGuid); if (entityOnDemandInfo == null) @@ -597,10 +597,8 @@ private static String getId(AtlasVertex vertex) { return vertex.getIdForDisplay(); } - private boolean incrementAndCheckIfRelationsLimitReached(AtlasEdge atlasEdge, boolean isInput, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, int depth, AtomicInteger entitiesTraversed, AtlasLineageOnDemandInfo.LineageDirection direction) { + private boolean incrementAndCheckIfRelationsLimitReached(AtlasEdge atlasEdge, boolean isInput, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, int depth, AtomicInteger entitiesTraversed, AtlasLineageOnDemandInfo.LineageDirection direction, Set visitedVertices) { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("incrementAndCheckIfRelationsLimitReached"); - if (lineageContainsVisitedEdgeV2(ret, atlasEdge)) - return false; AtlasVertex inVertex = isInput ? atlasEdge.getOutVertex() : atlasEdge.getInVertex(); String inGuid = AtlasGraphUtilsV2.getIdFromVertex(inVertex); @@ -613,7 +611,7 @@ private boolean incrementAndCheckIfRelationsLimitReached(AtlasEdge atlasEdge, bo LineageInfoOnDemand inLineageInfo = ret.getRelationsOnDemand().containsKey(inGuid) ? ret.getRelationsOnDemand().get(inGuid) : new LineageInfoOnDemand(inGuidLineageConstraints); LineageInfoOnDemand outLineageInfo = ret.getRelationsOnDemand().containsKey(outGuid) ? ret.getRelationsOnDemand().get(outGuid) : new LineageInfoOnDemand(outGuidLineageConstraints); - setHorizontalPaginationFlags(isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, inVertex, inGuid, outVertex, outGuid, inLineageInfo, outLineageInfo); + setHorizontalPaginationFlags(isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, inVertex, inGuid, outVertex, outGuid, inLineageInfo, outLineageInfo, visitedVertices); boolean hasRelationsLimitReached = setVerticalPaginationFlags(entitiesTraversed, inLineageInfo, outLineageInfo); if (!hasRelationsLimitReached) { @@ -640,9 +638,9 @@ private boolean setVerticalPaginationFlags(AtomicInteger entitiesTraversed, Line return hasRelationsLimitReached; } - private void setHorizontalPaginationFlags(boolean isInput, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, int depth, AtomicInteger entitiesTraversed, AtlasVertex inVertex, String inGuid, AtlasVertex outVertex, String outGuid, LineageInfoOnDemand inLineageInfo, LineageInfoOnDemand outLineageInfo) { - boolean isOutVertexVisited = ret.getRelationsOnDemand().containsKey(outGuid); - boolean isInVertexVisited = ret.getRelationsOnDemand().containsKey(inGuid); + private void setHorizontalPaginationFlags(boolean isInput, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, int depth, AtomicInteger entitiesTraversed, AtlasVertex inVertex, String inGuid, AtlasVertex outVertex, String outGuid, LineageInfoOnDemand inLineageInfo, LineageInfoOnDemand outLineageInfo, Set visitedVertices) { + boolean isOutVertexVisited = visitedVertices.contains(getId(outVertex)); + boolean isInVertexVisited = visitedVertices.contains(getId(inVertex)); if (depth == 1 || entitiesTraversed.get() == getLineageMaxNodeAllowedCount()-1) { // is the vertex a leaf? if (isInput && ! isOutVertexVisited) setHasUpstream(atlasLineageOnDemandContext, outVertex, outLineageInfo); @@ -652,24 +650,27 @@ else if (!isInput && ! isInVertexVisited) } private void setHasDownstream(AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasVertex inVertex, LineageInfoOnDemand inLineageInfo) { - List filteredEdges = getFilteredAtlasEdges(inVertex, PROCESS_INPUTS_EDGE, atlasLineageOnDemandContext); - if (!filteredEdges.isEmpty()) + List filteredEdges = getFilteredAtlasEdges(inVertex, IN, PROCESS_INPUTS_EDGE, atlasLineageOnDemandContext); + if (!filteredEdges.isEmpty()) { inLineageInfo.setHasDownstream(true); + inLineageInfo.setTotalOutputRelationsCount(filteredEdges.size()); + } } private void setHasUpstream(AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasVertex outVertex, LineageInfoOnDemand outLineageInfo) { - List filteredEdges = getFilteredAtlasEdges(outVertex, PROCESS_OUTPUTS_EDGE, atlasLineageOnDemandContext); - if (!filteredEdges.isEmpty()) + List filteredEdges = getFilteredAtlasEdges(outVertex, IN, PROCESS_OUTPUTS_EDGE, atlasLineageOnDemandContext); + if (!filteredEdges.isEmpty()) { outLineageInfo.setHasUpstream(true); + outLineageInfo.setTotalInputRelationsCount(filteredEdges.size()); + } } - private List getFilteredAtlasEdges(AtlasVertex outVertex, String processEdgeLabel, AtlasLineageOnDemandContext atlasLineageOnDemandContext) { + private List getFilteredAtlasEdges(AtlasVertex outVertex, AtlasEdgeDirection direction, String processEdgeLabel, AtlasLineageOnDemandContext atlasLineageOnDemandContext) { List filteredEdges = new ArrayList<>(); - Iterable edges = outVertex.getEdges(IN, processEdgeLabel); + Iterable edges = outVertex.getEdges(direction, processEdgeLabel); for (AtlasEdge edge : edges) { if (edgeMatchesEvaluation(edge, atlasLineageOnDemandContext)) { filteredEdges.add(edge); - break; } } return filteredEdges; diff --git a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java index 179d915df9..d5926e8a00 100755 --- a/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java +++ b/repository/src/main/java/org/apache/atlas/repository/graph/GraphHelper.java @@ -540,7 +540,8 @@ public static List getPropagatedVerticesIds (AtlasVertex classificationV } public static boolean hasEntityReferences(AtlasVertex classificationVertex) { - return classificationVertex.hasEdges(AtlasEdgeDirection.IN, CLASSIFICATION_LABEL); + Iterator edgeIterator = classificationVertex.query().direction(AtlasEdgeDirection.IN).label(CLASSIFICATION_LABEL).edges(1).iterator(); + return edgeIterator != null && edgeIterator.hasNext(); } public static List getAllPropagatedEntityVertices(AtlasVertex classificationVertex) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/aliasstore/ESAliasStore.java b/repository/src/main/java/org/apache/atlas/repository/store/aliasstore/ESAliasStore.java index ddd57e2664..2d272cb8fc 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/aliasstore/ESAliasStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/aliasstore/ESAliasStore.java @@ -65,6 +65,7 @@ @Component public class ESAliasStore implements IndexAliasStore { private static final Logger LOG = LoggerFactory.getLogger(ESAliasStore.class); + public static final String NEW_WILDCARD_DOMAIN_SUPER = "default/domain/*/super"; private final AtlasGraph graph; private final EntityGraphRetriever entityRetriever; @@ -214,7 +215,11 @@ private void personaPolicyToESDslClauses(List policies, } else if (getPolicyActions(policy).contains(ACCESS_READ_PERSONA_DOMAIN)) { for (String asset : assets) { - terms.add(asset); + if(!isAllDomain(asset)) { + terms.add(asset); + } else { + asset = NEW_WILDCARD_DOMAIN_SUPER; + } allowClauseList.add(mapOf("wildcard", mapOf(QUALIFIED_NAME, asset + "*"))); } @@ -246,6 +251,9 @@ private void personaPolicyToESDslClauses(List policies, allowClauseList.add(mapOf("terms", mapOf(QUALIFIED_NAME, terms))); } + private boolean isAllDomain(String asset) { + return asset.equals("*/super") || asset.equals("*") || asset.equals(NEW_WILDCARD_DOMAIN_SUPER); + } private Map esClausesToFilter(List> allowClauseList) { if (CollectionUtils.isNotEmpty(allowClauseList)) { return mapOf("bool", mapOf("should", allowClauseList)); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java index 596420696d..798d86071e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java @@ -3590,6 +3590,7 @@ public void updateClassifications(EntityMutationContext context, String guid, Li if (CollectionUtils.isEmpty(classifications)) { throw new AtlasBaseException(AtlasErrorCode.INVALID_CLASSIFICATION_PARAMS, "update", guid); } + entityRetriever.verifyClassificationsPropagationMode(classifications); AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(this.graph, guid); @@ -3711,7 +3712,21 @@ public void updateClassifications(EntityMutationContext context, String guid, Li Boolean updatedRestrictPropagationThroughLineage = classification.getRestrictPropagationThroughLineage(); Boolean currentRestrictPropagationThroughHierarchy = currentClassification.getRestrictPropagationThroughHierarchy(); Boolean updatedRestrictPropagationThroughHierarchy = classification.getRestrictPropagationThroughHierarchy(); - String propagationMode = entityRetriever.determinePropagationMode(updatedRestrictPropagationThroughLineage, updatedRestrictPropagationThroughHierarchy); + if (updatedRestrictPropagationThroughLineage == null) { + updatedRestrictPropagationThroughLineage = currentRestrictPropagationThroughLineage; + classification.setRestrictPropagationThroughLineage(updatedRestrictPropagationThroughLineage); + } + if (updatedRestrictPropagationThroughHierarchy == null) { + updatedRestrictPropagationThroughHierarchy = currentRestrictPropagationThroughHierarchy; + classification.setRestrictPropagationThroughHierarchy(updatedRestrictPropagationThroughHierarchy); + } + + String propagationMode = CLASSIFICATION_PROPAGATION_MODE_DEFAULT; + if (updatedTagPropagation) { + // determinePropagationMode also validates the propagation restriction option values + propagationMode = entityRetriever.determinePropagationMode(updatedRestrictPropagationThroughLineage, updatedRestrictPropagationThroughHierarchy); + } + if ((!Objects.equals(updatedRemovePropagations, currentRemovePropagations) || !Objects.equals(currentTagPropagation, updatedTagPropagation) || !Objects.equals(currentRestrictPropagationThroughLineage, updatedRestrictPropagationThroughLineage)) && @@ -3733,7 +3748,6 @@ public void updateClassifications(EntityMutationContext context, String guid, Li if (updatedTagPropagation) { if (updatedRestrictPropagationThroughLineage != null && !currentRestrictPropagationThroughLineage && updatedRestrictPropagationThroughLineage) { deleteDelegate.getHandler().removeTagPropagation(classificationVertex); - } if (updatedRestrictPropagationThroughHierarchy != null && !currentRestrictPropagationThroughHierarchy && updatedRestrictPropagationThroughHierarchy) { deleteDelegate.getHandler().removeTagPropagation(classificationVertex); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java index 62adf8119a..58fb516564 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/AuthPolicyPreProcessor.java @@ -42,10 +42,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.List; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; @@ -67,6 +64,7 @@ public class AuthPolicyPreProcessor implements PreProcessor { private static final Logger LOG = LoggerFactory.getLogger(AuthPolicyPreProcessor.class); + public static final String ENTITY_DEFAULT_DOMAIN_SUPER = "entity:default/domain/*/super"; private final AtlasGraph graph; private final AtlasTypeRegistry typeRegistry; @@ -127,6 +125,8 @@ private void processCreatePolicy(AtlasStruct entity) throws AtlasBaseException { if (!POLICY_SUB_CATEGORY_DOMAIN.equals(policySubCategory)) { validator.validate(policy, null, parentEntity, CREATE); validateConnectionAdmin(policy); + } else { + validateAndReduce(policy); } policy.setAttribute(QUALIFIED_NAME, String.format("%s/%s", getEntityQualifiedName(parentEntity), getUUID())); @@ -165,6 +165,21 @@ private void processCreatePolicy(AtlasStruct entity) throws AtlasBaseException { RequestContext.get().endMetricRecord(metricRecorder); } + + private void validateAndReduce(AtlasEntity policy) { + List resources = (List) policy.getAttribute(ATTR_POLICY_RESOURCES); + boolean hasAllDomainPattern = resources.stream().anyMatch(resource -> + resource.equals("entity:*") || + resource.equals("entity:*/super") || + resource.equals(ENTITY_DEFAULT_DOMAIN_SUPER) + ); + + if (hasAllDomainPattern) { + policy.setAttribute(ATTR_POLICY_RESOURCES, Collections.singletonList(ENTITY_DEFAULT_DOMAIN_SUPER)); + } + } + + private void processUpdatePolicy(AtlasStruct entity, AtlasVertex vertex) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdatePolicy"); AtlasEntity policy = (AtlasEntity) entity; @@ -182,6 +197,8 @@ private void processUpdatePolicy(AtlasStruct entity, AtlasVertex vertex) throws if (!POLICY_SUB_CATEGORY_DOMAIN.equals(policySubCategory)) { validator.validate(policy, existingPolicy, parentEntity, UPDATE); validateConnectionAdmin(policy); + } else { + validateAndReduce(policy); } String qName = getEntityQualifiedName(existingPolicy); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java index 1adbb8ec47..1cba29f935 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/accesscontrol/StakeholderPreProcessor.java @@ -58,8 +58,7 @@ import static org.apache.atlas.repository.Constants.STAKEHOLDER_ENTITY_TYPE; import static org.apache.atlas.repository.Constants.STAKEHOLDER_TITLE_ENTITY_TYPE; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.indexSearchPaginated; -import static org.apache.atlas.repository.store.graph.v2.preprocessor.datamesh.StakeholderTitlePreProcessor.ATTR_DOMAIN_QUALIFIED_NAMES; -import static org.apache.atlas.repository.store.graph.v2.preprocessor.datamesh.StakeholderTitlePreProcessor.STAR; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.datamesh.StakeholderTitlePreProcessor.*; import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_ACCESS_CONTROL_ENABLED; import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_PERSONA_ROLE_ID; import static org.apache.atlas.repository.util.AccessControlUtils.REL_ATTR_POLICIES; @@ -290,7 +289,7 @@ protected void ensureTitleAvailableForDomain(String domainQualifiedName, String List domainQualifiedNames = (List) stakeholderTitleHeader.getAttribute(ATTR_DOMAIN_QUALIFIED_NAMES); - if (!domainQualifiedNames.contains(STAR)) { + if (!domainQualifiedNames.contains(STAR) && !domainQualifiedNames.contains(NEW_STAR)) { Optional parentDomain = domainQualifiedNames.stream().filter(x -> domainQualifiedName.startsWith(x)).findFirst(); if (!parentDomain.isPresent()) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java index 0a4521e34b..a58108ce69 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/AbstractContractPreProcessor.java @@ -4,27 +4,29 @@ import org.apache.atlas.authorize.AtlasAuthorizationUtils; import org.apache.atlas.authorize.AtlasEntityAccessRequest; import org.apache.atlas.authorize.AtlasPrivilege; +import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.TypeCategory; +import org.apache.atlas.model.discovery.AtlasSearchResult; +import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.repository.graphdb.AtlasGraph; -import org.apache.atlas.repository.graphdb.AtlasVertex; -import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; import org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessor; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.utils.AtlasPerfMetrics; +import org.apache.commons.collections.CollectionUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.HashMap; -import java.util.Map; +import java.util.*; -import static org.apache.atlas.AtlasErrorCode.INSTANCE_BY_UNIQUE_ATTRIBUTE_NOT_FOUND; +import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST; import static org.apache.atlas.AtlasErrorCode.TYPE_NAME_INVALID; import static org.apache.atlas.repository.Constants.*; +import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; public abstract class AbstractContractPreProcessor implements PreProcessor { private static final Logger LOG = LoggerFactory.getLogger(AbstractContractPreProcessor.class); @@ -32,19 +34,21 @@ public abstract class AbstractContractPreProcessor implements PreProcessor { public final AtlasTypeRegistry typeRegistry; public final EntityGraphRetriever entityRetriever; public final AtlasGraph graph; + private final EntityDiscoveryService discovery; AbstractContractPreProcessor(AtlasGraph graph, AtlasTypeRegistry typeRegistry, - EntityGraphRetriever entityRetriever) { + EntityGraphRetriever entityRetriever, EntityDiscoveryService discovery) { this.graph = graph; this.typeRegistry = typeRegistry; this.entityRetriever = entityRetriever; + this.discovery = discovery; } - void authorizeContractCreateOrUpdate(AtlasEntity contractEntity, AtlasEntity.AtlasEntityWithExtInfo associatedAsset) throws AtlasBaseException { + void authorizeContractCreateOrUpdate(AtlasEntity contractEntity, AtlasEntity associatedAsset) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("authorizeContractUpdate"); try { - AtlasEntityHeader entityHeader = new AtlasEntityHeader(associatedAsset.getEntity()); + AtlasEntityHeader entityHeader = new AtlasEntityHeader(associatedAsset); //First authorize entity update access verifyAssetAccess(entityHeader, AtlasPrivilege.ENTITY_UPDATE, contractEntity, AtlasPrivilege.ENTITY_UPDATE); @@ -70,16 +74,39 @@ private void verifyAccess(AtlasEntityHeader entityHeader, AtlasPrivilege privile AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, privilege, entityHeader), errorMessage); } - AtlasEntity.AtlasEntityWithExtInfo getAssociatedAsset(String datasetQName, String typeName) throws AtlasBaseException { + public AtlasEntity getAssociatedAsset(String datasetQName, DataContract contract) throws AtlasBaseException { + IndexSearchParams indexSearchParams = new IndexSearchParams(); + Map dsl = new HashMap<>(); + int size = 2; + + List> mustClauseList = new ArrayList<>(); + mustClauseList.add(mapOf("term", mapOf(QUALIFIED_NAME, datasetQName))); + if (contract.getType() != null) { + mustClauseList.add(mapOf("term", mapOf("__typeName.keyword", contract.getType().name()))); + } else { + mustClauseList.add(mapOf("term", mapOf("__superTypeNames.keyword", SQL_ENTITY_TYPE))); + } + + dsl.put("query", mapOf("bool", mapOf("must", mustClauseList))); + dsl.put("sort", Collections.singletonList(mapOf(ATTR_CONTRACT_VERSION, mapOf("order", "desc")))); + dsl.put("size", size); - Map uniqAttributes = new HashMap<>(); - uniqAttributes.put(QUALIFIED_NAME, datasetQName); + indexSearchParams.setDsl(dsl); + indexSearchParams.setSuppressLogs(true); - AtlasEntityType entityType = ensureEntityType(typeName); + AtlasSearchResult result = discovery.directIndexSearch(indexSearchParams); + if (result == null || CollectionUtils.isEmpty(result.getEntities())) { + throw new AtlasBaseException("Dataset doesn't exist for given qualified name."); - AtlasVertex entityVertex = AtlasGraphUtilsV2.getVertexByUniqueAttributes(graph, entityType, uniqAttributes); + } else if (result.getEntities().size() >1 ) { + throw new AtlasBaseException(BAD_REQUEST, "Multiple dataset exists for given qualified name. " + + "Please specify the `type` attribute in contract."); + } else { + AtlasEntityHeader datasetEntity = result.getEntities().get(0); + contract.setType(datasetEntity.getTypeName()); + return new AtlasEntity(datasetEntity); + } - return entityRetriever.toAtlasEntityWithExtInfo(entityVertex); } AtlasEntityType ensureEntityType(String typeName) throws AtlasBaseException { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java index 1407c3c2ef..ac3df2535f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/ContractPreProcessor.java @@ -6,7 +6,6 @@ import org.apache.atlas.model.discovery.AtlasSearchResult; import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.instance.AtlasEntity; -import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; import org.apache.atlas.model.instance.AtlasStruct; import org.apache.atlas.model.instance.EntityMutations; import org.apache.atlas.repository.graphdb.AtlasGraph; @@ -52,7 +51,7 @@ public ContractPreProcessor(AtlasGraph graph, AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, boolean storeDifferentialAudits, EntityDiscoveryService discovery) { - super(graph, typeRegistry, entityRetriever); + super(graph, typeRegistry, entityRetriever, discovery); this.storeDifferentialAudits = storeDifferentialAudits; this.discovery = discovery; this.entityComparator = new AtlasEntityComparator(typeRegistry, entityRetriever, null, true, true); @@ -79,14 +78,9 @@ private void processUpdateContract(AtlasEntity entity, EntityMutationContext con AtlasEntity existingContractEntity = entityRetriever.toAtlasEntity(vertex); // No update to relationships allowed for the existing contract version resetAllRelationshipAttributes(entity); - DataContract contract = DataContract.deserialize(contractString); - String existingContractString = getContractString(existingContractEntity); - boolean requestFromMigration = RequestContext.get().getRequestContextHeaders().getOrDefault( - "x-atlan-request-id", "").contains("json-to-yaml-migration"); - if (!requestFromMigration && !StringUtils.isEmpty(contractString) && - !contract.equals(DataContract.deserialize(existingContractString))) { + if (existingContractEntity.getAttribute(ATTR_CERTIFICATE_STATUS) == DataContract.Status.VERIFIED.name()) { // Update the same asset(entity) - throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Can't update a specific version of contract"); + throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Can't update published version of contract."); } } private void processCreateContract(AtlasEntity entity, EntityMutationContext context) throws AtlasBaseException { @@ -109,8 +103,8 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con String contractString = getContractString(entity); DataContract contract = DataContract.deserialize(contractString); String datasetQName = contractQName.substring(0, contractQName.lastIndexOf('/')); - contractQName = String.format("%s/%s/%s", datasetQName, contract.getType().name(), CONTRACT_QUALIFIED_NAME_SUFFIX); - AtlasEntityWithExtInfo associatedAsset = getAssociatedAsset(datasetQName, contract.getType().name()); + AtlasEntity associatedAsset = getAssociatedAsset(datasetQName, contract); + contractQName = String.format("%s/%s/%s", datasetQName, associatedAsset.getTypeName(), CONTRACT_QUALIFIED_NAME_SUFFIX); authorizeContractCreateOrUpdate(entity, associatedAsset); @@ -120,8 +114,11 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con String contractStringJSON = DataContract.serializeJSON(contract); entity.setAttribute(ATTR_CONTRACT_JSON, contractStringJSON); - AtlasEntity currentVersionEntity = getCurrentVersion(associatedAsset.getEntity().getGuid()); + AtlasEntity currentVersionEntity = getCurrentVersion(associatedAsset.getGuid()); Long newVersionNumber = 1L; + if (currentVersionEntity == null && contract.getStatus() == DataContract.Status.VERIFIED) { + throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Can't create a new published version"); + } if (currentVersionEntity != null) { // Contract already exist Long currentVersionNumber = (Long) currentVersionEntity.getAttribute(ATTR_CONTRACT_VERSION); @@ -130,13 +127,16 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con // No changes in the contract, Not creating new version removeCreatingVertex(context, entity); return; - } else if (contract.equals(DataContract.deserialize(getContractString(currentVersionEntity)))) { + } else if (!currentVersionEntity.getAttribute(ATTR_CERTIFICATE_STATUS).equals(DataContract.Status.VERIFIED.name())) { resetAllRelationshipAttributes(entity); - // No change in contract, metadata changed + // Contract is in draft state. Update the same version updateExistingVersion(context, entity, currentVersionEntity); newVersionNumber = currentVersionNumber; } else { - // contract changed (metadata might/not changed). Create new version. + // Current version is published. Creating a new draft version. + if (contract.getStatus() == DataContract.Status.VERIFIED) { + throw new AtlasBaseException(OPERATION_NOT_SUPPORTED, "Can't create a new published version"); + } newVersionNumber = currentVersionNumber + 1; resetAllRelationshipAttributes(entity); @@ -150,9 +150,9 @@ private void processCreateContract(AtlasEntity entity, EntityMutationContext con } entity.setAttribute(QUALIFIED_NAME, String.format("%s/V%s", contractQName, newVersionNumber)); entity.setAttribute(ATTR_CONTRACT_VERSION, newVersionNumber); - entity.setAttribute(ATTR_ASSET_GUID, associatedAsset.getEntity().getGuid()); + entity.setAttribute(ATTR_ASSET_GUID, associatedAsset.getGuid()); - datasetAttributeSync(context, associatedAsset.getEntity(), entity); + datasetAttributeSync(context, associatedAsset, entity); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java index 4ceea2853c..81f5c24da5 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/contract/DataContract.java @@ -37,14 +37,12 @@ public class DataContract { @Valid @NotNull public String kind; - public Status status; - @JsonProperty(value = "template_version", defaultValue = "0.0.1") - public String templateVersion; - @Valid @NotNull + public Status status = Status.DRAFT; + @JsonProperty(value = "template_version") + public String templateVersion = "0.0.1"; public String data_source; @Valid @NotNull public String dataset; - @Valid @NotNull public DatasetType type; public String description; public List owners; @@ -89,7 +87,7 @@ public static DatasetType from(String s) throws AtlasBaseException { case "materialisedview": return MaterialisedView; default: - throw new AtlasBaseException(String.format("dataset.type: %s value not supported yet.", s)); + throw new AtlasBaseException(BAD_REQUEST, String.format("type: %s value not supported yet.", s)); } } } @@ -144,7 +142,7 @@ public void setType(String type) throws AtlasBaseException { try { this.type = DatasetType.from(type); } catch (IllegalArgumentException | AtlasBaseException ex) { - throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "type " + type + " is inappropriate. Accepted values: " + Arrays.toString(DatasetType.values())); + throw new AtlasBaseException(AtlasErrorCode.INVALID_VALUE, "type: " + type + " is inappropriate. Accepted values: " + Arrays.toString(DatasetType.values())); } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java index 97685d1ed6..1de5fe685e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/StakeholderTitlePreProcessor.java @@ -48,6 +48,7 @@ public class StakeholderTitlePreProcessor implements PreProcessor { public static final String STAR = "*/super"; + public static final String NEW_STAR = "default/domain/*/super"; public static final String ATTR_DOMAIN_QUALIFIED_NAMES = "stakeholderTitleDomainQualifiedNames"; public static final String REL_ATTR_STAKEHOLDERS = "stakeholders"; @@ -114,13 +115,13 @@ private void processCreateStakeholderTitle(AtlasEntity entity) throws AtlasBaseE if (CollectionUtils.isEmpty(domainQualifiedNames)) { throw new AtlasBaseException(BAD_REQUEST, "Please provide attribute " + ATTR_DOMAIN_QUALIFIED_NAMES); } - - if (domainQualifiedNames.contains(STAR)) { + if (domainQualifiedNames.contains(NEW_STAR) || domainQualifiedNames.contains(STAR)) { if (domainQualifiedNames.size() > 1) { - domainQualifiedNames.clear(); - domainQualifiedNames.add(STAR); + domainQualifiedNames.add(NEW_STAR); entity.setAttribute(ATTR_DOMAIN_QUALIFIED_NAMES, domainQualifiedNames); + }else { + domainQualifiedNames.replaceAll(s -> s.equals(STAR) ? NEW_STAR : s); } String qualifiedName = format(PATTERN_QUALIFIED_NAME_ALL_DOMAINS, getUUID()); @@ -211,8 +212,8 @@ private void authorizeDomainAccess(List domainQualifiedNames) throws Atl for (String domainQualifiedName: domainQualifiedNames) { String domainQualifiedNameToAuth; - if (domainQualifiedNames.contains(STAR)) { - domainQualifiedNameToAuth = "*/super"; + if (domainQualifiedNames.contains(STAR) || domainQualifiedNames.contains(NEW_STAR)) { + domainQualifiedNameToAuth = NEW_STAR; } else { domainQualifiedNameToAuth = domainQualifiedName; }