Skip to content

Commit

Permalink
Merge pull request #3365 from atlanhq/disc-367-glossary-filter
Browse files Browse the repository at this point in the history 
DISC-367 : Update Glossary policy alias filtering query
  • Loading branch information
@sumandas0
sumandas0 authored Jul 24, 2024
2 parents 39144b8 + 9915fba commit 976baa3
Show file tree
Hide file tree
Showing 14 changed files with 144 additions and 94 deletions.
10 changes: 5 additions & 5 deletions .github/workflows/chart-release-dispatcher.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ jobs:
# Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
- uses: actions/checkout@v3
with:
token: ${{ secrets.my_pat }}
token: ${{ secrets.ORG_PAT_GITHUB }}
ref: ${{ steps.extract_branch.outputs.branch }}
fetch-depth: 0

Expand All @@ -50,10 +50,10 @@ jobs:
- name: Get PR url and PR User
id: get_pr_url_user
run: |
head_sha=$(curl -s -H "Authorization: Bearer ${{ secrets.my_pat }}" -H "Accept: application/vnd.github.v3+json" "https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }}/jobs" | jq -r '.jobs[0].head_sha')
head_sha=$(curl -s -H "Authorization: Bearer ${{ secrets.ORG_PAT_GITHUB }}" -H "Accept: application/vnd.github.v3+json" "https://api.github.com/repos/${{ github.repository }}/actions/runs/${{ github.event.workflow_run.id }}/jobs" | jq -r '.jobs[0].head_sha')
echo "Head SHA: $head_sha"
pr_url=$(curl -s -H "Authorization: Bearer ${{ secrets.my_pat }}" -H "Accept: application/vnd.github.v3+json" "https://api.github.com/search/issues?q=sha:$head_sha+type:pr" | jq -r '.items[0].html_url')
pr_user=$(curl -s -H "Authorization: Bearer ${{ secrets.my_pat }}" -H "Accept: application/vnd.github.v3+json" "https://api.github.com/search/issues?q=sha:$head_sha+type:pr" | jq -r '.items[0].user.login')
pr_url=$(curl -s -H "Authorization: Bearer ${{ secrets.ORG_PAT_GITHUB }}" -H "Accept: application/vnd.github.v3+json" "https://api.github.com/search/issues?q=sha:$head_sha+type:pr" | jq -r '.items[0].html_url')
pr_user=$(curl -s -H "Authorization: Bearer ${{ secrets.ORG_PAT_GITHUB }}" -H "Accept: application/vnd.github.v3+json" "https://api.github.com/search/issues?q=sha:$head_sha+type:pr" | jq -r '.items[0].user.login')
echo "pr_url=$pr_url" >> $GITHUB_OUTPUT
echo "pr_user=$pr_user" >> $GITHUB_OUTPUT
Expand All @@ -65,7 +65,7 @@ jobs:
- name: Repository Dispatch
uses: peter-evans/repository-dispatch@v2
with:
token: ${{ secrets.my_pat }}
token: ${{ secrets.ORG_PAT_GITHUB }}
repository: ${{ matrix.repo }}
event-type: dispatch_chart_release_workflow
client-payload: |-
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/main-ecr.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,4 +196,4 @@ jobs:
${{ steps.login-ecr.outputs.registry }}/atlanhq/${{ github.event.repository.name }}:${{ steps.get_branch.outputs.branch }}-${{ steps.semver_tag.outputs.new_tag }}
build-args: |
ACCESS_TOKEN_USR=$GITHUB_ACTOR
ACCESS_TOKEN_PWD=${{ secrets.my_pat }}
ACCESS_TOKEN_PWD=${{ secrets.ORG_PAT_GITHUB }}
6 changes: 3 additions & 3 deletions .github/workflows/maven.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,7 +57,7 @@ jobs:
[{
"id": "github",
"username": "atlan-ci",
"password": "${{ secrets.my_pat }}"
"password": "${{ secrets.ORG_PAT_GITHUB }}"
}]
- name: Build with Maven
Expand All @@ -76,7 +76,7 @@ jobs:
shell: bash

- name: Get version tag
run: echo "##[set-output name=version;]$(echo `git ls-remote https://${{ secrets.my_pat }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ steps.get_branch.outputs.branch }} | awk '{ print $1}' | cut -c1-7`)abcd"
run: echo "##[set-output name=version;]$(echo `git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ steps.get_branch.outputs.branch }} | awk '{ print $1}' | cut -c1-7`)abcd"
id: get_version

- name: Set up Buildx
Expand All @@ -88,7 +88,7 @@ jobs:
with:
registry: ghcr.io
username: $GITHUB_ACTOR
password: ${{ secrets.my_pat }}
password: ${{ secrets.ORG_PAT_GITHUB }}

- name: Build and push
id: docker_build
Expand Down
2 changes: 1 addition & 1 deletion .github/workflows/trivy-docker-scan.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ jobs:
output: 'trivy-results-docker.sarif'
exit-code: '1'
#ignore-unfixed: true
severity: 'CRITICAL,HIGH,MEDIUM'
severity: 'CRITICAL,HIGH'

- name: Upload Trivy Docker Scan Results To GitHub Security tab
uses: github/codeql-action/upload-sarif@v2
Expand Down
2 changes: 2 additions & 0 deletions common/src/main/java/org/apache/atlas/repository/Constants.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,8 @@ public final class Constants {
/**
* SQL property keys.
*/

public static final String SQL_ENTITY_TYPE = "SQL";
public static final String CONNECTION_ENTITY_TYPE = "Connection";
public static final String QUERY_ENTITY_TYPE = "Query";
public static final String QUERY_FOLDER_ENTITY_TYPE = "Folder";
Expand Down
22 changes: 19 additions & 3 deletions intg/src/main/java/org/apache/atlas/model/lineage/AtlasLineageOnDemandInfo.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,6 +171,8 @@ public static class LineageInfoOnDemand {
boolean hasMoreOutputs;
int inputRelationsCount;
int outputRelationsCount;
int totalInputRelationsCount;
int totalOutputRelationsCount;
boolean isInputRelationsReachedLimit;
boolean isOutputRelationsReachedLimit;
@JsonProperty
Expand All @@ -188,13 +190,15 @@ public LineageInfoOnDemand(LineageOnDemandConstraints onDemandConstraints) {
this.hasMoreOutputs = false;
this.inputRelationsCount = 0;
this.outputRelationsCount = 0;
this.totalInputRelationsCount = 0;
this.totalOutputRelationsCount = 0;
this.isInputRelationsReachedLimit = false;
this.isOutputRelationsReachedLimit = false;
this.hasUpstream = false;
this.hasDownstream = false;
this.fromCounter = 0;
}

public boolean isInputRelationsReachedLimit() {
return isInputRelationsReachedLimit;
}
Expand Down Expand Up @@ -243,10 +247,18 @@ public void setHasDownstream(boolean hasDownstream) {
this.hasDownstream = hasDownstream;
}

public int getFromCounter() {
return fromCounter;
public int getTotalInputRelationsCount() {
return totalInputRelationsCount;
}

public void setTotalInputRelationsCount(int count) {this.totalInputRelationsCount = count;}

public int getTotalOutputRelationsCount() {
return totalOutputRelationsCount;
}

public void setTotalOutputRelationsCount(int count) {this.totalOutputRelationsCount = count;}

public void incrementFromCounter() {
fromCounter++;
}
Expand All @@ -255,6 +267,10 @@ public int getInputRelationsCount() {
return inputRelationsCount;
}

public int getFromCounter() {
return fromCounter;
}

public void incrementInputRelationsCount() {
this.inputRelationsCount++;
if (inputRelationsCount == onDemandConstraints.getInputRelationsLimit()) {
Expand Down
35 changes: 18 additions & 17 deletions repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -343,7 +343,7 @@ private void traverseEdgesOnDemand(Iterator<AtlasEdge> processEdges, boolean isI
}

boolean isInputEdge = processEdge.getLabel().equalsIgnoreCase(PROCESS_INPUTS_EDGE);
if (incrementAndCheckIfRelationsLimitReached(processEdge, isInputEdge, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, direction)) {
if (incrementAndCheckIfRelationsLimitReached(processEdge, isInputEdge, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, direction, new HashSet<>())) {
break;
} else {
addEdgeToResult(processEdge, ret, atlasLineageOnDemandContext, nextLevel, traversalOrder);
Expand Down Expand Up @@ -387,7 +387,7 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i
continue;
}

if (incrementAndCheckIfRelationsLimitReached(incomingEdge, !isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, direction)) {
if (incrementAndCheckIfRelationsLimitReached(incomingEdge, !isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, direction, visitedVertices)) {
LineageInfoOnDemand entityOnDemandInfo = ret.getRelationsOnDemand().get(baseGuid);
if (entityOnDemandInfo == null)
continue;
Expand All @@ -414,7 +414,7 @@ private void traverseEdgesOnDemand(AtlasVertex datasetVertex, boolean isInput, i
if (checkForOffset(outgoingEdge, processVertex, atlasLineageOnDemandContext, ret)) {
continue;
}
if (incrementAndCheckIfRelationsLimitReached(outgoingEdge, isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, direction)) {
if (incrementAndCheckIfRelationsLimitReached(outgoingEdge, isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, direction, visitedVertices)) {
String processGuid = AtlasGraphUtilsV2.getIdFromVertex(processVertex);
LineageInfoOnDemand entityOnDemandInfo = ret.getRelationsOnDemand().get(processGuid);
if (entityOnDemandInfo == null)
Expand Down Expand Up @@ -597,10 +597,8 @@ private static String getId(AtlasVertex vertex) {
return vertex.getIdForDisplay();
}

private boolean incrementAndCheckIfRelationsLimitReached(AtlasEdge atlasEdge, boolean isInput, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, int depth, AtomicInteger entitiesTraversed, AtlasLineageOnDemandInfo.LineageDirection direction) {
private boolean incrementAndCheckIfRelationsLimitReached(AtlasEdge atlasEdge, boolean isInput, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, int depth, AtomicInteger entitiesTraversed, AtlasLineageOnDemandInfo.LineageDirection direction, Set<String> visitedVertices) {
AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("incrementAndCheckIfRelationsLimitReached");
if (lineageContainsVisitedEdgeV2(ret, atlasEdge))
return false;

AtlasVertex inVertex = isInput ? atlasEdge.getOutVertex() : atlasEdge.getInVertex();
String inGuid = AtlasGraphUtilsV2.getIdFromVertex(inVertex);
Expand All @@ -613,7 +611,7 @@ private boolean incrementAndCheckIfRelationsLimitReached(AtlasEdge atlasEdge, bo
LineageInfoOnDemand inLineageInfo = ret.getRelationsOnDemand().containsKey(inGuid) ? ret.getRelationsOnDemand().get(inGuid) : new LineageInfoOnDemand(inGuidLineageConstraints);
LineageInfoOnDemand outLineageInfo = ret.getRelationsOnDemand().containsKey(outGuid) ? ret.getRelationsOnDemand().get(outGuid) : new LineageInfoOnDemand(outGuidLineageConstraints);

setHorizontalPaginationFlags(isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, inVertex, inGuid, outVertex, outGuid, inLineageInfo, outLineageInfo);
setHorizontalPaginationFlags(isInput, atlasLineageOnDemandContext, ret, depth, entitiesTraversed, inVertex, inGuid, outVertex, outGuid, inLineageInfo, outLineageInfo, visitedVertices);

boolean hasRelationsLimitReached = setVerticalPaginationFlags(entitiesTraversed, inLineageInfo, outLineageInfo);
if (!hasRelationsLimitReached) {
Expand All @@ -640,9 +638,9 @@ private boolean setVerticalPaginationFlags(AtomicInteger entitiesTraversed, Line
return hasRelationsLimitReached;
}

private void setHorizontalPaginationFlags(boolean isInput, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, int depth, AtomicInteger entitiesTraversed, AtlasVertex inVertex, String inGuid, AtlasVertex outVertex, String outGuid, LineageInfoOnDemand inLineageInfo, LineageInfoOnDemand outLineageInfo) {
boolean isOutVertexVisited = ret.getRelationsOnDemand().containsKey(outGuid);
boolean isInVertexVisited = ret.getRelationsOnDemand().containsKey(inGuid);
private void setHorizontalPaginationFlags(boolean isInput, AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasLineageOnDemandInfo ret, int depth, AtomicInteger entitiesTraversed, AtlasVertex inVertex, String inGuid, AtlasVertex outVertex, String outGuid, LineageInfoOnDemand inLineageInfo, LineageInfoOnDemand outLineageInfo, Set<String> visitedVertices) {
boolean isOutVertexVisited = visitedVertices.contains(getId(outVertex));
boolean isInVertexVisited = visitedVertices.contains(getId(inVertex));
if (depth == 1 || entitiesTraversed.get() == getLineageMaxNodeAllowedCount()-1) { // is the vertex a leaf?
if (isInput && ! isOutVertexVisited)
setHasUpstream(atlasLineageOnDemandContext, outVertex, outLineageInfo);
Expand All @@ -652,24 +650,27 @@ else if (!isInput && ! isInVertexVisited)
}

private void setHasDownstream(AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasVertex inVertex, LineageInfoOnDemand inLineageInfo) {
List<AtlasEdge> filteredEdges = getFilteredAtlasEdges(inVertex, PROCESS_INPUTS_EDGE, atlasLineageOnDemandContext);
if (!filteredEdges.isEmpty())
List<AtlasEdge> filteredEdges = getFilteredAtlasEdges(inVertex, IN, PROCESS_INPUTS_EDGE, atlasLineageOnDemandContext);
if (!filteredEdges.isEmpty()) {
inLineageInfo.setHasDownstream(true);
inLineageInfo.setTotalOutputRelationsCount(filteredEdges.size());
}
}

private void setHasUpstream(AtlasLineageOnDemandContext atlasLineageOnDemandContext, AtlasVertex outVertex, LineageInfoOnDemand outLineageInfo) {
List<AtlasEdge> filteredEdges = getFilteredAtlasEdges(outVertex, PROCESS_OUTPUTS_EDGE, atlasLineageOnDemandContext);
if (!filteredEdges.isEmpty())
List<AtlasEdge> filteredEdges = getFilteredAtlasEdges(outVertex, IN, PROCESS_OUTPUTS_EDGE, atlasLineageOnDemandContext);
if (!filteredEdges.isEmpty()) {
outLineageInfo.setHasUpstream(true);
outLineageInfo.setTotalInputRelationsCount(filteredEdges.size());
}
}

private List<AtlasEdge> getFilteredAtlasEdges(AtlasVertex outVertex, String processEdgeLabel, AtlasLineageOnDemandContext atlasLineageOnDemandContext) {
private List<AtlasEdge> getFilteredAtlasEdges(AtlasVertex outVertex, AtlasEdgeDirection direction, String processEdgeLabel, AtlasLineageOnDemandContext atlasLineageOnDemandContext) {
List<AtlasEdge> filteredEdges = new ArrayList<>();
Iterable<AtlasEdge> edges = outVertex.getEdges(IN, processEdgeLabel);
Iterable<AtlasEdge> edges = outVertex.getEdges(direction, processEdgeLabel);
for (AtlasEdge edge : edges) {
if (edgeMatchesEvaluation(edge, atlasLineageOnDemandContext)) {
filteredEdges.add(edge);
break;
}
}
return filteredEdges;
Expand Down
10 changes: 5 additions & 5 deletions repository/src/main/java/org/apache/atlas/repository/store/aliasstore/ESAliasStore.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@

import javax.inject.Inject;
import java.util.*;
import java.util.stream.Collectors;

import static org.apache.atlas.ESAliasRequestBuilder.ESAliasAction.ADD;
import static org.apache.atlas.repository.Constants.PERSONA_ENTITY_TYPE;
Expand All @@ -60,6 +61,7 @@
import static org.apache.atlas.repository.util.AccessControlUtils.getPolicyConnectionQN;
import static org.apache.atlas.repository.util.AccessControlUtils.getPurposeTags;
import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf;
import static org.apache.atlas.type.Constants.GLOSSARY_PROPERTY_KEY;


@Component
Expand Down Expand Up @@ -207,13 +209,11 @@ private void personaPolicyToESDslClauses(List<AtlasEntity> policies,
terms.add(connectionQName);

} else if (getPolicyActions(policy).contains(ACCESS_READ_PERSONA_GLOSSARY)) {

for (String glossaryQName : assets) {
terms.add(glossaryQName);
allowClauseList.add(mapOf("wildcard", mapOf(QUALIFIED_NAME, "*@" + glossaryQName)));
if (CollectionUtils.isNotEmpty(assets)) {
terms.addAll(assets);
allowClauseList.add(mapOf("terms", mapOf(GLOSSARY_PROPERTY_KEY, assets)));
}
} else if (getPolicyActions(policy).contains(ACCESS_READ_PERSONA_DOMAIN)) {

for (String asset : assets) {
if(!isAllDomain(asset)) {
terms.add(asset);
Expand Down
18 changes: 16 additions & 2 deletions repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphMapper.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3590,6 +3590,7 @@ public void updateClassifications(EntityMutationContext context, String guid, Li
if (CollectionUtils.isEmpty(classifications)) {
throw new AtlasBaseException(AtlasErrorCode.INVALID_CLASSIFICATION_PARAMS, "update", guid);
}
entityRetriever.verifyClassificationsPropagationMode(classifications);

AtlasVertex entityVertex = AtlasGraphUtilsV2.findByGuid(this.graph, guid);

Expand Down Expand Up @@ -3711,7 +3712,21 @@ public void updateClassifications(EntityMutationContext context, String guid, Li
Boolean updatedRestrictPropagationThroughLineage = classification.getRestrictPropagationThroughLineage();
Boolean currentRestrictPropagationThroughHierarchy = currentClassification.getRestrictPropagationThroughHierarchy();
Boolean updatedRestrictPropagationThroughHierarchy = classification.getRestrictPropagationThroughHierarchy();
String propagationMode = entityRetriever.determinePropagationMode(updatedRestrictPropagationThroughLineage, updatedRestrictPropagationThroughHierarchy);
if (updatedRestrictPropagationThroughLineage == null) {
updatedRestrictPropagationThroughLineage = currentRestrictPropagationThroughLineage;
classification.setRestrictPropagationThroughLineage(updatedRestrictPropagationThroughLineage);
}
if (updatedRestrictPropagationThroughHierarchy == null) {
updatedRestrictPropagationThroughHierarchy = currentRestrictPropagationThroughHierarchy;
classification.setRestrictPropagationThroughHierarchy(updatedRestrictPropagationThroughHierarchy);
}

String propagationMode = CLASSIFICATION_PROPAGATION_MODE_DEFAULT;
if (updatedTagPropagation) {
// determinePropagationMode also validates the propagation restriction option values
propagationMode = entityRetriever.determinePropagationMode(updatedRestrictPropagationThroughLineage, updatedRestrictPropagationThroughHierarchy);
}

if ((!Objects.equals(updatedRemovePropagations, currentRemovePropagations) ||
!Objects.equals(currentTagPropagation, updatedTagPropagation) ||
!Objects.equals(currentRestrictPropagationThroughLineage, updatedRestrictPropagationThroughLineage)) &&
Expand All @@ -3733,7 +3748,6 @@ public void updateClassifications(EntityMutationContext context, String guid, Li
if (updatedTagPropagation) {
if (updatedRestrictPropagationThroughLineage != null && !currentRestrictPropagationThroughLineage && updatedRestrictPropagationThroughLineage) {
deleteDelegate.getHandler().removeTagPropagation(classificationVertex);

}
if (updatedRestrictPropagationThroughHierarchy != null && !currentRestrictPropagationThroughHierarchy && updatedRestrictPropagationThroughHierarchy) {
deleteDelegate.getHandler().removeTagPropagation(classificationVertex);
Expand Down
Loading

0 comments on commit 976baa3

Please sign in to comment.