From f38a52a47801c0930516cfda632b1a330cf33061 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Tue, 4 Jun 2024 11:48:36 +0530 Subject: [PATCH 1/6] feat: rebuild alias endpoint --- .../store/aliasstore/ESAliasStore.java | 28 +++++++++++++++++++ .../store/graph/AtlasEntityStore.java | 2 ++ .../store/graph/v2/AtlasEntityStoreV2.java | 10 +++++++ .../org/apache/atlas/web/rest/EntityREST.java | 7 +++++ 4 files changed, 47 insertions(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/aliasstore/ESAliasStore.java b/repository/src/main/java/org/apache/atlas/repository/store/aliasstore/ESAliasStore.java index 1991106b96..88fdf0f4af 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/aliasstore/ESAliasStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/aliasstore/ESAliasStore.java @@ -141,6 +141,23 @@ public boolean updateAlias(AtlasEntity.AtlasEntityWithExtInfo accessControl, Atl return true; } + public void rebuildAlias(AtlasEntity.AtlasEntityWithExtInfo accessControl) throws AtlasBaseException { + String aliasName = getAliasName(accessControl.getEntity()); + + Map filter; + + if (PERSONA_ENTITY_TYPE.equals(accessControl.getEntity().getTypeName())) { + filter = getFilterForPersona(accessControl); + } else { + filter = getFilterForPurpose(accessControl.getEntity()); + } + + ESAliasRequestBuilder requestBuilder = new ESAliasRequestBuilder(); + requestBuilder.addAction(ADD, new AliasAction(getIndexNameFromAliasIfExists(VERTEX_INDEX_NAME), aliasName, filter)); + + graph.createOrUpdateESAlias(requestBuilder); + } + @Override public boolean deleteAlias(String aliasName) throws AtlasBaseException { graph.deleteESAlias(getIndexNameFromAliasIfExists(VERTEX_INDEX_NAME), aliasName); @@ -161,6 +178,17 @@ private Map getFilterForPersona(AtlasEntity.AtlasEntityWithExtIn return esClausesToFilter(allowClauseList); } + private Map getFilterForPersona(AtlasEntity.AtlasEntityWithExtInfo persona) throws AtlasBaseException { + List> allowClauseList = new ArrayList<>(); + + List policies = getPolicies(persona); + if (CollectionUtils.isNotEmpty(policies)) { + personaPolicyToESDslClauses(policies, allowClauseList); + } + + return esClausesToFilter(allowClauseList); + } + private Map getFilterForPurpose(AtlasEntity purpose) throws AtlasBaseException { List> allowClauseList = new ArrayList<>(); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java index 960ec3808e..d072139aef 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java @@ -364,4 +364,6 @@ EntityMutationResponse deleteByUniqueAttributes(List objectIds) void repairMeaningAttributeForTerms(List termGuids) throws AtlasBaseException; + public void repairAlias(String guid) throws AtlasBaseException; + } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 2d6d28c0a7..77d530307e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -49,6 +49,7 @@ import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.patches.PatchContext; import org.apache.atlas.repository.patches.ReIndexPatch; +import org.apache.atlas.repository.store.aliasstore.ESAliasStore; import org.apache.atlas.repository.store.graph.AtlasEntityStore; import org.apache.atlas.repository.store.graph.AtlasRelationshipStore; import org.apache.atlas.repository.store.graph.EntityGraphDiscovery; @@ -147,6 +148,8 @@ public class AtlasEntityStoreV2 implements AtlasEntityStore { private final AtlasRelationshipStore atlasRelationshipStore; private final FeatureFlagStore featureFlagStore; + private final ESAliasStore esAliasStore; + @Inject public AtlasEntityStoreV2(AtlasGraph graph, DeleteHandlerDelegate deleteDelegate, RestoreHandlerV1 restoreHandlerV1, AtlasTypeRegistry typeRegistry, IAtlasEntityChangeNotifier entityChangeNotifier, EntityGraphMapper entityGraphMapper, TaskManagement taskManagement, @@ -163,6 +166,7 @@ public AtlasEntityStoreV2(AtlasGraph graph, DeleteHandlerDelegate deleteDelegate this.taskManagement = taskManagement; this.atlasRelationshipStore = atlasRelationshipStore; this.featureFlagStore = featureFlagStore; + this.esAliasStore = new ESAliasStore(graph, entityRetriever); try { this.discovery = new EntityDiscoveryService(typeRegistry, graph, null, null, null, null); @@ -2703,6 +2707,12 @@ private void repairMeanings(AtlasVertex assetVertex) { } } + @Override + public void repairAlias(String guid) throws AtlasBaseException { + // Fetch entity with extenfo + AtlasEntity.AtlasEntityWithExtInfo entity = entityRetriever.toAtlasEntityWithExtInfo(guid); + this.esAliasStore.rebuildAlias(entity); + } } diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java index 0ebb48b16c..31283924a2 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java @@ -1930,4 +1930,11 @@ public void repairIndexByTypeName(@PathParam("typename") String typename, @Query AtlasPerfTracer.log(perf); } } + + @POST + @Path("/repairalias/{guid}") + @Timed + public void repairAlias(@PathParam("guid") String guid) throws AtlasBaseException { + entitiesStore.repairAlias(guid); + } } From 3cd1e03b860a8f3b4af1ea36e9e1f13b50675041 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Tue, 4 Jun 2024 19:16:35 +0530 Subject: [PATCH 2/6] feat: add more checks --- .../store/graph/v2/AtlasEntityStoreV2.java | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 77d530307e..846510c9ea 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -2709,8 +2709,25 @@ private void repairMeanings(AtlasVertex assetVertex) { } @Override public void repairAlias(String guid) throws AtlasBaseException { - // Fetch entity with extenfo + // Fetch entity with extInfo AtlasEntity.AtlasEntityWithExtInfo entity = entityRetriever.toAtlasEntityWithExtInfo(guid); + + if (entity == null) { + throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guid); + } + + // Validate entity status + if (entity.getEntity().getStatus() != ACTIVE) { + throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_DELETED, guid); + } + + // Validate entity type + String entityType = entity.getEntity().getTypeName(); + if (!PERSONA_ENTITY_TYPE.equals(entityType)) { + throw new AtlasBaseException(AtlasErrorCode.INVALID_OBJECT_ID, entityType); + } + + // Rebuild alias this.esAliasStore.rebuildAlias(entity); } } From 269d78c9a1b62318696a8301eac12ca44ffd8515 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Wed, 5 Jun 2024 08:50:41 +0530 Subject: [PATCH 3/6] fix: add fix to include referred entities --- .../store/graph/v2/AtlasEntityStoreV2.java | 27 +++++++++++++------ 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 846510c9ea..7df859eb99 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -116,6 +116,7 @@ import static org.apache.atlas.repository.graph.GraphHelper.getStatus; import static org.apache.atlas.repository.store.graph.v2.EntityGraphMapper.validateLabels; import static org.apache.atlas.repository.store.graph.v2.tasks.MeaningsTaskFactory.*; +import static org.apache.atlas.repository.util.AccessControlUtils.REL_ATTR_POLICIES; import static org.apache.atlas.type.Constants.HAS_LINEAGE; import static org.apache.atlas.type.Constants.HAS_LINEAGE_VALID; import static org.apache.atlas.type.Constants.MEANINGS_TEXT_PROPERTY_KEY; @@ -2709,26 +2710,36 @@ private void repairMeanings(AtlasVertex assetVertex) { } @Override public void repairAlias(String guid) throws AtlasBaseException { - // Fetch entity with extInfo - AtlasEntity.AtlasEntityWithExtInfo entity = entityRetriever.toAtlasEntityWithExtInfo(guid); + AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("repairAlias"); + // Fetch accesscontrolEntity with extInfo + AtlasEntity.AtlasEntityWithExtInfo accesscontrolEntity = entityRetriever.toAtlasEntityWithExtInfo(guid); - if (entity == null) { + AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, new AtlasEntityHeader(accesscontrolEntity.getEntity()))); + + if (accesscontrolEntity == null) { throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guid); } - // Validate entity status - if (entity.getEntity().getStatus() != ACTIVE) { + // Validate accesscontrolEntity status + if (accesscontrolEntity.getEntity().getStatus() != ACTIVE) { throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_DELETED, guid); } - // Validate entity type - String entityType = entity.getEntity().getTypeName(); + // Validate accesscontrolEntity type + String entityType = accesscontrolEntity.getEntity().getTypeName(); if (!PERSONA_ENTITY_TYPE.equals(entityType)) { throw new AtlasBaseException(AtlasErrorCode.INVALID_OBJECT_ID, entityType); } + List policies = (List) accesscontrolEntity.getEntity().getRelationshipAttribute(REL_ATTR_POLICIES); + for (AtlasObjectId policy : policies) { + accesscontrolEntity.addReferredEntity(entityRetriever.toAtlasEntity(policy)); + } + // Rebuild alias - this.esAliasStore.rebuildAlias(entity); + this.esAliasStore.rebuildAlias(accesscontrolEntity); + + RequestContext.get().endMetricRecord(metric); } } From a0294805d2feddf40d33f286e0fad670b58c90b8 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Wed, 5 Jun 2024 09:54:27 +0530 Subject: [PATCH 4/6] nit: make all namings descriptive --- .../store/graph/AtlasEntityStore.java | 3 +-- .../store/graph/v2/AtlasEntityStoreV2.java | 2 +- .../org/apache/atlas/web/rest/EntityREST.java | 24 ++++++++++++++++--- 3 files changed, 23 insertions(+), 6 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java index d072139aef..912799cdd6 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/AtlasEntityStore.java @@ -25,7 +25,6 @@ import org.apache.atlas.model.instance.AtlasEntity.AtlasEntitiesWithExtInfo; import org.apache.atlas.model.instance.AtlasEntity.AtlasEntityWithExtInfo; import org.apache.atlas.model.instance.AtlasEntityHeader; -import org.apache.atlas.model.instance.AtlasEntityHeaders; import org.apache.atlas.model.instance.AtlasObjectId; import org.apache.atlas.model.instance.AtlasHasLineageRequests; import org.apache.atlas.model.instance.EntityMutationResponse; @@ -364,6 +363,6 @@ EntityMutationResponse deleteByUniqueAttributes(List objectIds) void repairMeaningAttributeForTerms(List termGuids) throws AtlasBaseException; - public void repairAlias(String guid) throws AtlasBaseException; + void repairAccesscontrolAlias(String guid) throws AtlasBaseException; } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 7df859eb99..037fadbc17 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -2709,7 +2709,7 @@ private void repairMeanings(AtlasVertex assetVertex) { } @Override - public void repairAlias(String guid) throws AtlasBaseException { + public void repairAccesscontrolAlias(String guid) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metric = RequestContext.get().startMetricRecord("repairAlias"); // Fetch accesscontrolEntity with extInfo AtlasEntity.AtlasEntityWithExtInfo accesscontrolEntity = entityRetriever.toAtlasEntityWithExtInfo(guid); diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java index 31283924a2..7669b07839 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java @@ -1932,9 +1932,27 @@ public void repairIndexByTypeName(@PathParam("typename") String typename, @Query } @POST - @Path("/repairalias/{guid}") + @Path("/repairAccesscontrolAlias/{guid}") @Timed - public void repairAlias(@PathParam("guid") String guid) throws AtlasBaseException { - entitiesStore.repairAlias(guid); + public void repairAccessControlAlias(@PathParam("guid") String guid) throws AtlasBaseException { + Servlets.validateQueryParamLength("guid", guid); + + AtlasPerfTracer perf = null; + + + try { + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "EntityREST.repairAccessControlAlias"); + } + + entitiesStore.repairAccesscontrolAlias(guid); + + LOG.info("Repaired access control alias for entity with guid {}", guid); + + } finally { + AtlasPerfTracer.log(perf); + } + + } } From b9b38aebaf519c07185dfb5df191556519aaad7c Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Wed, 5 Jun 2024 13:22:57 +0530 Subject: [PATCH 5/6] fix: remove unwanted funcs --- .../store/aliasstore/ESAliasStore.java | 28 ------------------- .../store/graph/v2/AtlasEntityStoreV2.java | 8 ++---- 2 files changed, 2 insertions(+), 34 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/aliasstore/ESAliasStore.java b/repository/src/main/java/org/apache/atlas/repository/store/aliasstore/ESAliasStore.java index 88fdf0f4af..1991106b96 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/aliasstore/ESAliasStore.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/aliasstore/ESAliasStore.java @@ -141,23 +141,6 @@ public boolean updateAlias(AtlasEntity.AtlasEntityWithExtInfo accessControl, Atl return true; } - public void rebuildAlias(AtlasEntity.AtlasEntityWithExtInfo accessControl) throws AtlasBaseException { - String aliasName = getAliasName(accessControl.getEntity()); - - Map filter; - - if (PERSONA_ENTITY_TYPE.equals(accessControl.getEntity().getTypeName())) { - filter = getFilterForPersona(accessControl); - } else { - filter = getFilterForPurpose(accessControl.getEntity()); - } - - ESAliasRequestBuilder requestBuilder = new ESAliasRequestBuilder(); - requestBuilder.addAction(ADD, new AliasAction(getIndexNameFromAliasIfExists(VERTEX_INDEX_NAME), aliasName, filter)); - - graph.createOrUpdateESAlias(requestBuilder); - } - @Override public boolean deleteAlias(String aliasName) throws AtlasBaseException { graph.deleteESAlias(getIndexNameFromAliasIfExists(VERTEX_INDEX_NAME), aliasName); @@ -178,17 +161,6 @@ private Map getFilterForPersona(AtlasEntity.AtlasEntityWithExtIn return esClausesToFilter(allowClauseList); } - private Map getFilterForPersona(AtlasEntity.AtlasEntityWithExtInfo persona) throws AtlasBaseException { - List> allowClauseList = new ArrayList<>(); - - List policies = getPolicies(persona); - if (CollectionUtils.isNotEmpty(policies)) { - personaPolicyToESDslClauses(policies, allowClauseList); - } - - return esClausesToFilter(allowClauseList); - } - private Map getFilterForPurpose(AtlasEntity purpose) throws AtlasBaseException { List> allowClauseList = new ArrayList<>(); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 037fadbc17..83df0fe2ef 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -2716,10 +2716,6 @@ public void repairAccesscontrolAlias(String guid) throws AtlasBaseException { AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_UPDATE, new AtlasEntityHeader(accesscontrolEntity.getEntity()))); - if (accesscontrolEntity == null) { - throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_NOT_FOUND, guid); - } - // Validate accesscontrolEntity status if (accesscontrolEntity.getEntity().getStatus() != ACTIVE) { throw new AtlasBaseException(AtlasErrorCode.INSTANCE_GUID_DELETED, guid); @@ -2728,7 +2724,7 @@ public void repairAccesscontrolAlias(String guid) throws AtlasBaseException { // Validate accesscontrolEntity type String entityType = accesscontrolEntity.getEntity().getTypeName(); if (!PERSONA_ENTITY_TYPE.equals(entityType)) { - throw new AtlasBaseException(AtlasErrorCode.INVALID_OBJECT_ID, entityType); + throw new AtlasBaseException(AtlasErrorCode.OPERATION_NOT_SUPPORTED, entityType); } List policies = (List) accesscontrolEntity.getEntity().getRelationshipAttribute(REL_ATTR_POLICIES); @@ -2737,7 +2733,7 @@ public void repairAccesscontrolAlias(String guid) throws AtlasBaseException { } // Rebuild alias - this.esAliasStore.rebuildAlias(accesscontrolEntity); + this.esAliasStore.updateAlias(accesscontrolEntity, null); RequestContext.get().endMetricRecord(metric); } From dbf011bc321480a760f014c7560ae16d2b0cd729 Mon Sep 17 00:00:00 2001 From: Suman Das <59254445+sumandas0@users.noreply.github.com> Date: Wed, 5 Jun 2024 13:23:47 +0530 Subject: [PATCH 6/6] nit: have a good api path --- webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java index 7669b07839..3aadc06ad2 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/EntityREST.java @@ -1932,7 +1932,7 @@ public void repairIndexByTypeName(@PathParam("typename") String typename, @Query } @POST - @Path("/repairAccesscontrolAlias/{guid}") + @Path("/repair/accesscontrolAlias/{guid}") @Timed public void repairAccessControlAlias(@PathParam("guid") String guid) throws AtlasBaseException { Servlets.validateQueryParamLength("guid", guid);