Sync beta with master

.github/workflows/maven.yml

@@ -1,3 +1,4 @@
+
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -26,6 +27,7 @@ on:
       - development
       - master
       - lineageondemand
+      - janus0.6
 
 jobs:
   build:
@@ -34,12 +36,12 @@ jobs:
 
     steps:
       - uses: actions/checkout@v2
-      
+
       - name: Set up JDK 1.8
         uses: actions/setup-java@v1
         with:
           java-version: 1.8
-      
+
       - name: Cache Maven packages
         uses: actions/cache@v2
         with:
@@ -48,8 +50,9 @@ jobs:
           restore-keys: ${{ runner.os }}-m2
 
       - name: Get branch name
-        run: echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
-        id: get_branch
+        run: |
+          echo "BRANCH_NAME=${GITHUB_REF#refs/heads/}" >> $GITHUB_ENV
+          echo BRANCH_NAME=${GITHUB_REF#refs/heads/}
 
       - name: Create Maven Settings
         uses: s4u/[email protected]
@@ -63,7 +66,7 @@ jobs:
 
       - name: Build with Maven
         run: |
-          branch_name=${{ steps.get_branch.outputs.branch }} 
+          branch_name=${{ env.BRANCH_NAME }}
           if [[ $branch_name == 'main' || $branch_name == 'master' || $branch_name == 'lineageondemand' ]]
           then
               echo "build without dashboard"
@@ -73,19 +76,26 @@ jobs:
               chmod +x ./build.sh && ./build.sh
           fi
 
-      - run:   echo "REPOSITORY_NAME=`echo "$GITHUB_REPOSITORY" | awk -F / '{print $2}' | sed -e "s/:refs//"`" >> $GITHUB_ENV
+      - name: Get Repository Name
+        run:   echo "REPOSITORY_NAME=`echo "$GITHUB_REPOSITORY" | awk -F / '{print $2}' | sed -e "s/:refs//"`" >> $GITHUB_ENV
         shell: bash
 
       - name: Get version tag
-        run: echo "##[set-output name=version;]$(echo `git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ steps.get_branch.outputs.branch }} | awk '{ print $1}' | cut -c1-7`)abcd"
-        id: get_version
+        # run: echo "##[set-output name=version;]$(echo `git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ env.BRANCH_NAME }} | awk '{ print $1}' | cut -c1-7`)abcd"
+        run: |
+          echo "VERSION=$(git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ env.BRANCH_NAME }} | awk '{ print $1}' | cut -c1-7 | head -n 1)abcd"
+          echo "VERSION=$(git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ env.BRANCH_NAME }} | awk '{ print $1}' | cut -c1-7 | tr -d '[:space:]')abcd"
+          echo "VERSION=$(git ls-remote https://${{ secrets.ORG_PAT_GITHUB }}@github.com/atlanhq/${REPOSITORY_NAME}.git ${{ env.BRANCH_NAME }} | awk '{ print $1}' | cut -c1-7 | tr -d '[:space:]')abcd" >> $GITHUB_ENV
+
+      - name: Get commit ID
+        run: echo "COMMIT_ID=$(echo ${GITHUB_SHA} | cut -c1-7)abcd" >> $GITHUB_ENV
 
-      - name: Set up Buildx 
+      - name: Set up Buildx
         id: buildx
         uses: docker/setup-buildx-action@v1
 
       - name: Login to GitHub Registry
-        uses: docker/login-action@v1 
+        uses: docker/login-action@v1
         with:
           registry: ghcr.io
           username: $GITHUB_ACTOR
@@ -102,8 +112,8 @@ jobs:
           provenance: true
           push: true
           tags: |
-            ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch }}:latest
-            ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ steps.get_branch.outputs.branch }}:${{ steps.get_version.outputs.version }}
+            ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ env.BRANCH_NAME }}:latest
+            ghcr.io/atlanhq/${{ github.event.repository.name }}-${{ env.BRANCH_NAME }}:${{ env.COMMIT_ID }}
 
       - name: Scan Image
         uses: aquasecurity/trivy-action@master
@@ -116,4 +126,4 @@ jobs:
       - name: Upload Trivy scan results to GitHub Security tab
         uses: github/codeql-action/[email protected]
         with:
-          sarif_file: 'trivy-image-results.sarif'
+          sarif_file: 'trivy-image-results.sarif'

common/src/main/java/org/apache/atlas/repository/Constants.java

@@ -39,6 +39,7 @@
 import java.util.Set;
 
 import static org.apache.atlas.type.AtlasStructType.AtlasAttribute.encodePropertyKey;
+import static org.apache.atlas.type.AtlasStructType.UNIQUE_ATTRIBUTE_SHADE_PROPERTY_PREFIX;
 
 /**
  * Repository Constants.
@@ -269,6 +270,7 @@ public final class Constants {
 
     public static final String NAME                                    = "name";
     public static final String QUALIFIED_NAME                          = "qualifiedName";
+    public static final String UNIQUE_QUALIFIED_NAME                   = UNIQUE_ATTRIBUTE_SHADE_PROPERTY_PREFIX + QUALIFIED_NAME;
     public static final String TYPE_NAME_PROPERTY_KEY                  = INTERNAL_PROPERTY_KEY_PREFIX + "typeName";
     public static final String INDEX_SEARCH_MAX_RESULT_SET_SIZE        = "atlas.graph.index.search.max-result-set-size";
     public static final String INDEX_SEARCH_TYPES_MAX_QUERY_STR_LENGTH = "atlas.graph.index.search.types.max-query-str-length";

repository/src/main/java/org/apache/atlas/discovery/EntityLineageService.java

@@ -534,15 +534,15 @@ private void traverseEdgesUsingBFS(String baseGuid, AtlasLineageListContext line
     private void enqueueNeighbours(AtlasVertex currentVertex, boolean isDataset,
                                    AtlasLineageListContext lineageListContext, Queue<String> traversalQueue,
                                    Set<String> visitedVertices, Set<String> skippedVertices,
-                                   Map<String, List<String>> lineageParentsForEntityMap, Map<String, List<String>> lineageChildrenForEntityMap) {
+                                   Map<String, List<String>> lineageParentsForEntityMap, Map<String, List<String>> lineageChildrenForEntityMap) throws AtlasBaseException {
         AtlasPerfMetrics.MetricRecorder traverseEdgesOnDemandGetEdges = RequestContext.get().startMetricRecord("traverseEdgesOnDemandGetEdges");
         Iterator<AtlasEdge> edges;
         if (isDataset)
-            edges = currentVertex.getEdges(IN, isInputDirection(lineageListContext) ? PROCESS_OUTPUTS_EDGE : PROCESS_INPUTS_EDGE).iterator();
+            edges = GraphHelper.getActiveEdges(currentVertex, isInputDirection(lineageListContext) ? PROCESS_OUTPUTS_EDGE : PROCESS_INPUTS_EDGE, IN);
         else
-            edges = currentVertex.getEdges(OUT, isInputDirection(lineageListContext) ? PROCESS_INPUTS_EDGE : PROCESS_OUTPUTS_EDGE).iterator();
-        RequestContext.get().endMetricRecord(traverseEdgesOnDemandGetEdges);
+            edges = GraphHelper.getActiveEdges(currentVertex, isInputDirection(lineageListContext) ? PROCESS_INPUTS_EDGE : PROCESS_OUTPUTS_EDGE, OUT);
 
+        RequestContext.get().endMetricRecord(traverseEdgesOnDemandGetEdges);
         while (edges.hasNext()) {
             AtlasEdge currentEdge = edges.next();
             if (!lineageListContext.evaluateTraversalFilter(currentEdge))

repository/src/main/java/org/apache/atlas/repository/store/graph/v2/EntityGraphRetriever.java

@@ -1043,7 +1043,6 @@ private Map<String, Object> preloadProperties(AtlasVertex entityVertex, AtlasEnt
                     if (typeCategory == TypeCategory.ARRAY && elementTypeCategory == TypeCategory.PRIMITIVE) {
                         updateAttrValue(propertiesMap, property);
                     } else if (attribute == null && isMultiValuedProperty) {
-                        LOG.warn("Multiple value found for key {} for entity vertex: {}", property.key(), entityVertex);
                         updateAttrValue(propertiesMap, property);
                     } else if (propertiesMap.get(property.key()) == null) {
                         propertiesMap.put(property.key(), property.value());
@@ -1081,7 +1080,12 @@ private boolean isPolicyAttribute(Set<String> attributes) {
                 AccessControlUtils.ATTR_POLICY_RESOURCES_CATEGORY,
                 AccessControlUtils.ATTR_POLICY_SERVICE_NAME,
                 AccessControlUtils.ATTR_POLICY_PRIORITY,
-                AccessControlUtils.REL_ATTR_POLICIES));
+                AccessControlUtils.REL_ATTR_POLICIES,
+                AccessControlUtils.ATTR_SERVICE_SERVICE_TYPE,
+                AccessControlUtils.ATTR_SERVICE_TAG_SERVICE,
+                AccessControlUtils.ATTR_SERVICE_IS_ENABLED,
+                AccessControlUtils.ATTR_SERVICE_LAST_SYNC)
+        );
 
         return exclusionSet.stream().anyMatch(attributes::contains);
     }
@@ -1926,11 +1930,7 @@ public Object getVertexAttributePreFetchCache(AtlasVertex vertex, AtlasAttribute
 
         // value is present as marker, fetch the value from the vertex
         if (ATLAS_INDEXSEARCH_ENABLE_FETCHING_NON_PRIMITIVE_ATTRIBUTES.getBoolean()) {
-            //AtlasPerfMetrics.MetricRecorder nonPrimitiveAttributes = RequestContext.get().startMetricRecord("processNonPrimitiveAttributes");
             return mapVertexToAttribute(vertex, attribute, null, false);
-            //LOG.debug("capturing excluded property set category and value, mapVertexValue - {}: {} : {} : {}", attribute.getName(), attribute.getAttributeType().getTypeCategory(), properties.get(attribute.getName()), mappedVertex);
-            //RequestContext.get().endMetricRecord(nonPrimitiveAttributes);
-            //return mappedVertex;
         }
 
         return null;

repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqueQNAttributeMigrationService.java

@@ -0,0 +1,99 @@
+package org.apache.atlas.repository.store.graph.v2;
+
+import org.apache.atlas.exception.AtlasBaseException;
+import org.apache.atlas.repository.graph.GraphHelper;
+import org.apache.atlas.repository.graphdb.AtlasVertex;
+import org.apache.commons.collections.CollectionUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.*;
+
+public class UniqueQNAttributeMigrationService {
+
+    private static final Logger LOG = LoggerFactory.getLogger(UniqueQNAttributeMigrationService.class);
+
+    private final EntityGraphRetriever entityRetriever;
+
+
+    private Set<String> entityGuids;
+    private final TransactionInterceptHelper   transactionInterceptHelper;
+
+    private final String QUALIFIED_NAME_ATTR = "qualifiedName";
+    private final String UNIQUE_QUALIFIED_NAME_ATTR = "__u_qualifiedName";
+
+    public UniqueQNAttributeMigrationService(EntityGraphRetriever entityRetriever, Set<String> entityGuids, TransactionInterceptHelper transactionInterceptHelper) {
+        this.entityRetriever = entityRetriever;
+        this.transactionInterceptHelper = transactionInterceptHelper;
+        this.entityGuids = entityGuids;
+    }
+
+    public void migrateQN() throws Exception {
+        try {
+            int count = 0;
+            int totalUpdatedCount = 0;
+            for (String entityGuid : entityGuids) {
+                AtlasVertex entityVertex = entityRetriever.getEntityVertex(entityGuid);
+
+                if (entityVertex == null) {
+                    LOG.error("Entity vertex not found for guid: {}", entityGuid);
+                    continue;
+                }
+
+                boolean isCommitRequired = migrateuniqueQnAttr(entityVertex);
+                if (isCommitRequired){
+                    count++;
+                    totalUpdatedCount++;
+                }
+                else {
+                    LOG.info("No changes to commit for entity: {} as no migration needed", entityGuid);
+                }
+
+                if (count == 20) {
+                    LOG.info("Committing batch of 20 entities...");
+                    commitChanges();
+                    count = 0;
+                }
+            }
+
+            if (count > 0) {
+                LOG.info("Committing remaining {} entities...", count);
+                commitChanges();
+            }
+
+            LOG.info("Total Vertex updated: {}", totalUpdatedCount);
+
+        } catch (Exception e) {
+            LOG.error("Error while migration unique qualifiedName attribute for entities: {}", entityGuids, e);
+            throw e;
+        }
+    }
+
+    private boolean migrateuniqueQnAttr(AtlasVertex vertex) throws AtlasBaseException {
+        try{
+            boolean isCommitRequired = false;
+
+            String qualifiedName = vertex.getProperty(QUALIFIED_NAME_ATTR, String.class);
+            String uniqueQualifiedName = vertex.getProperty(UNIQUE_QUALIFIED_NAME_ATTR, String.class);
+
+            if(!qualifiedName.equals(uniqueQualifiedName)) {
+                vertex.setProperty(UNIQUE_QUALIFIED_NAME_ATTR, qualifiedName);
+                isCommitRequired = true;
+            }
+            return isCommitRequired;
+        }catch (Exception e) {
+            LOG.error("Failed to migrate unique qualifiedName attribute for entity: ", e);
+            throw e;
+        }
+    }
+
+    public void commitChanges() throws AtlasBaseException {
+        try {
+            transactionInterceptHelper.intercept();
+            LOG.info("Committed a entity to the graph");
+        } catch (Exception e){
+            LOG.error("Failed to commit asset: ", e);
+            throw e;
+        }
+    }
+}

repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/datamesh/DataDomainPreProcessor.java

@@ -294,6 +294,9 @@ private void moveChildrenToAnotherDomain(AtlasVertex childDomainVertex,
             childDomainVertex.setProperty(QUALIFIED_NAME, updatedDomainQualifiedName);
             updatedAttributes.put(QUALIFIED_NAME, updatedDomainQualifiedName);
 
+            // Change unique qualifiedName attribute
+            childDomainVertex.setProperty(UNIQUE_QUALIFIED_NAME, updatedDomainQualifiedName);
+
             //change superDomainQN, parentDomainQN
             childDomainVertex.setProperty(SUPER_DOMAIN_QN_ATTR, superDomainQualifiedName);
             childDomainVertex.setProperty(PARENT_DOMAIN_QN_ATTR, parentDomainQualifiedName);
@@ -348,6 +351,7 @@ private void moveChildDataProductToAnotherDomain(AtlasVertex productVertex,
             String updatedQualifiedName = parentDomainQualifiedName + getOwnQualifiedNameForChild(currentQualifiedName);
 
             productVertex.setProperty(QUALIFIED_NAME, updatedQualifiedName);
+            productVertex.setProperty(UNIQUE_QUALIFIED_NAME, updatedQualifiedName);
             updatedAttributes.put(QUALIFIED_NAME, updatedQualifiedName);
 
             productVertex.setProperty(PARENT_DOMAIN_QN_ATTR, parentDomainQualifiedName);

repository/src/main/java/org/apache/atlas/repository/util/AccessControlUtils.java

@@ -61,6 +61,10 @@ public final class AccessControlUtils {
 
     public static final String ATTR_PURPOSE_CLASSIFICATIONS  = "purposeClassifications";
 
+    public static final String ATTR_SERVICE_SERVICE_TYPE = "authServiceType";
+    public static final String ATTR_SERVICE_TAG_SERVICE  = "tagService";
+    public static final String ATTR_SERVICE_IS_ENABLED   = "authServiceIsEnabled";
+    public static final String ATTR_SERVICE_LAST_SYNC    = "authServicePolicyLastSync";
     public static final String ATTR_POLICY_TYPE  = "policyType";
     public static final String ATTR_POLICY_USERS  = "policyUsers";
     public static final String ATTR_POLICY_GROUPS  = "policyGroups";

webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java

@@ -325,6 +325,31 @@ public List<AtlasEntity> searchForType(@PathParam("typeName") String typeName, @
         }
     }
 
+    @POST
+    @Path("repair-unique-qualified-name")
+    @Timed
+    public Boolean updateUniqueQualifiedName(final Set<String> assetGuids) throws Exception {
+        AtlasPerfTracer perf = null;
+        try {
+            if (CollectionUtils.isEmpty(assetGuids)) {
+                throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Asset GUIDs are required for which updating unique qualified name is required");
+            }
+
+            if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) {
+                perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "MigrationREST.updateUniqueQualifiedName(" + assetGuids + ")");
+            }
+
+            UniqueQNAttributeMigrationService migrationService = new UniqueQNAttributeMigrationService(entityRetriever, assetGuids, transactionInterceptHelper);
+            migrationService.migrateQN();
+        } catch (Exception e) {
+            LOG.error("Error while updating unique qualified name for guids: {}", assetGuids, e);
+            throw e;
+        } finally {
+            AtlasPerfTracer.log(perf);
+        }
+        return Boolean.TRUE;
+    }
+
     private List<AtlasEntity> getEntitiesByIndexSearch(IndexSearchParams indexSearchParams, Boolean minExtInfo, boolean ignoreRelationships) throws AtlasBaseException {
         List<AtlasEntity> entities = new ArrayList<>();
         String indexName = "janusgraph_vertex_index";