From e48975a2f4899b338ca69b44e15d660111f85b90 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Fri, 6 Dec 2024 15:52:28 +0530 Subject: [PATCH 1/4] mesh-294: migration API to migrate unique qn attribute --- .../instance/UniqueQnMigrationRequest.java | 66 +++++++++++++++ .../v2/UniqueQNAttributeMigrationService.java | 84 +++++++++++++++++++ .../apache/atlas/web/rest/MigrationREST.java | 26 ++++++ 3 files changed, 176 insertions(+) create mode 100644 intg/src/main/java/org/apache/atlas/model/instance/UniqueQnMigrationRequest.java create mode 100644 repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqueQNAttributeMigrationService.java diff --git a/intg/src/main/java/org/apache/atlas/model/instance/UniqueQnMigrationRequest.java b/intg/src/main/java/org/apache/atlas/model/instance/UniqueQnMigrationRequest.java new file mode 100644 index 0000000000..df263eb761 --- /dev/null +++ b/intg/src/main/java/org/apache/atlas/model/instance/UniqueQnMigrationRequest.java @@ -0,0 +1,66 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.atlas.model.instance; + +import com.fasterxml.jackson.annotation.JsonAutoDetect; +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.databind.annotation.JsonSerialize; + +import javax.xml.bind.annotation.XmlAccessType; +import javax.xml.bind.annotation.XmlAccessorType; +import javax.xml.bind.annotation.XmlRootElement; +import java.io.Serializable; +import java.util.Set; + +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; +import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; + +/** + * Request to link/unlink policies from asset. + */ +@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE) +@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) +@JsonIgnoreProperties(ignoreUnknown = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +@XmlRootElement +@XmlAccessorType(XmlAccessType.PROPERTY) +public class UniqueQnMigrationRequest implements Serializable { + private static final long serialVersionUID = 1L; + + private Set assetGuids; + + + public Set getAssetGuids() { + return assetGuids; + } + + public void setAssetGuids(Set assetGuids) { + this.assetGuids = assetGuids; + } + + + @Override + public String toString() { + final StringBuilder sb = new StringBuilder("LinkBusinessPolicyRequest{"); + sb.append("assetGuids=").append(assetGuids); + sb.append('}'); + return sb.toString(); + } +} + diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqueQNAttributeMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqueQNAttributeMigrationService.java new file mode 100644 index 0000000000..47e61b2b38 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqueQNAttributeMigrationService.java @@ -0,0 +1,84 @@ +package org.apache.atlas.repository.store.graph.v2; + +import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.repository.graph.GraphHelper; +import org.apache.atlas.repository.graphdb.AtlasVertex; +import org.apache.commons.collections.CollectionUtils; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.util.*; + +public class UniqueQNAttributeMigrationService { + + private static final Logger LOG = LoggerFactory.getLogger(UniqueQNAttributeMigrationService.class); + + private final EntityGraphRetriever entityRetriever; + + + private Set entityGuids; + private final TransactionInterceptHelper transactionInterceptHelper; + + private final String QUALIFIED_NAME_ATTR = "qualifiedName"; + private final String UNIQUE_QUALIFIED_NAME_ATTR = "__u_qualifiedName"; + + public UniqueQNAttributeMigrationService(EntityGraphRetriever entityRetriever, Set entityGuids, TransactionInterceptHelper transactionInterceptHelper) { + this.entityRetriever = entityRetriever; + this.transactionInterceptHelper = transactionInterceptHelper; + this.entityGuids = entityGuids; + } + + public void migrateQN() throws Exception { + try { + for (String entityGuid : entityGuids) { + AtlasVertex entityVertex = entityRetriever.getEntityVertex(entityGuid); + + if (entityVertex == null) { + LOG.error("Entity vertex not found for guid: {}", entityGuid); + continue; + } + + boolean isCommitRequired = migrateuniqueQnAttr(entityVertex); + if (isCommitRequired){ + LOG.info("Committing changes for entity: {}", entityGuid); + commitChanges(); + } + else { + LOG.info("No changes to commit for entity: {} as no migration needed", entityGuid); + } + } + + } catch (Exception e) { + LOG.error("Error while migration unique qualifiedName attribute for entities: {}", entityGuids, e); + throw e; + } + } + + private boolean migrateuniqueQnAttr(AtlasVertex vertex) throws AtlasBaseException { + try{ + boolean isCommitRequired = false; + + String qualifiedName = vertex.getProperty(QUALIFIED_NAME_ATTR, String.class); + String uniqueQualifiedName = vertex.getProperty(UNIQUE_QUALIFIED_NAME_ATTR, String.class); + + if(!qualifiedName.equals(uniqueQualifiedName)) { + vertex.setProperty(UNIQUE_QUALIFIED_NAME_ATTR, qualifiedName); + isCommitRequired = true; + } + return isCommitRequired; + }catch (Exception e) { + LOG.error("Failed to migrate unique qualifiedName attribute for entity: ", e); + throw e; + } + } + + public void commitChanges() throws AtlasBaseException { + try { + transactionInterceptHelper.intercept(); + LOG.info("Committed a entity to the graph"); + } catch (Exception e){ + LOG.error("Failed to commit asset: ", e); + throw e; + } + } +} \ No newline at end of file diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java index 551d0f4aa2..a480793cf1 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java @@ -8,6 +8,7 @@ import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.EntityMutationResponse; +import org.apache.atlas.model.instance.UniqueQnMigrationRequest; import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graphdb.*; import org.apache.atlas.repository.store.graph.AtlasEntityStore; @@ -325,6 +326,31 @@ public List searchForType(@PathParam("typeName") String typeName, @ } } + @POST + @Path("update-unique-qualified-name") + @Timed + public Boolean updateUniqueQualifiedName(final UniqueQnMigrationRequest request) throws Exception { + AtlasPerfTracer perf = null; + try { + if (CollectionUtils.isEmpty(request.getAssetGuids())) { + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Asset GUIDs are required for which updating unique qualified name is required"); + } + + if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "MigrationREST.updateUniqueQualifiedName(" + request.getAssetGuids() + ")"); + } + + UniqueQNAttributeMigrationService migrationService = new UniqueQNAttributeMigrationService(entityRetriever, request.getAssetGuids(), transactionInterceptHelper); + migrationService.migrateQN(); + } catch (Exception e) { + LOG.error("Error while updating unique qualified name for guids: {}", request.getAssetGuids(), e); + throw e; + } finally { + AtlasPerfTracer.log(perf); + } + return Boolean.TRUE; + } + private List getEntitiesByIndexSearch(IndexSearchParams indexSearchParams, Boolean minExtInfo, boolean ignoreRelationships) throws AtlasBaseException { List entities = new ArrayList<>(); String indexName = "janusgraph_vertex_index"; From f4e02273f7c1ccbfb8fccb2fa61634535921e74f Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Wed, 11 Dec 2024 08:18:03 +0530 Subject: [PATCH 2/4] mesh-294: resolved PR comments --- .../instance/UniqueQnMigrationRequest.java | 66 ------------------- .../v2/UniqueQNAttributeMigrationService.java | 11 +++- .../apache/atlas/web/rest/MigrationREST.java | 13 ++-- 3 files changed, 15 insertions(+), 75 deletions(-) delete mode 100644 intg/src/main/java/org/apache/atlas/model/instance/UniqueQnMigrationRequest.java diff --git a/intg/src/main/java/org/apache/atlas/model/instance/UniqueQnMigrationRequest.java b/intg/src/main/java/org/apache/atlas/model/instance/UniqueQnMigrationRequest.java deleted file mode 100644 index df263eb761..0000000000 --- a/intg/src/main/java/org/apache/atlas/model/instance/UniqueQnMigrationRequest.java +++ /dev/null @@ -1,66 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - *

- * http://www.apache.org/licenses/LICENSE-2.0 - *

- * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.atlas.model.instance; - -import com.fasterxml.jackson.annotation.JsonAutoDetect; -import com.fasterxml.jackson.annotation.JsonIgnoreProperties; -import com.fasterxml.jackson.annotation.JsonInclude; -import com.fasterxml.jackson.databind.annotation.JsonSerialize; - -import javax.xml.bind.annotation.XmlAccessType; -import javax.xml.bind.annotation.XmlAccessorType; -import javax.xml.bind.annotation.XmlRootElement; -import java.io.Serializable; -import java.util.Set; - -import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.NONE; -import static com.fasterxml.jackson.annotation.JsonAutoDetect.Visibility.PUBLIC_ONLY; - -/** - * Request to link/unlink policies from asset. - */ -@JsonAutoDetect(getterVisibility = PUBLIC_ONLY, setterVisibility = PUBLIC_ONLY, fieldVisibility = NONE) -@JsonSerialize(include = JsonSerialize.Inclusion.NON_NULL) -@JsonIgnoreProperties(ignoreUnknown = true) -@JsonInclude(JsonInclude.Include.NON_NULL) -@XmlRootElement -@XmlAccessorType(XmlAccessType.PROPERTY) -public class UniqueQnMigrationRequest implements Serializable { - private static final long serialVersionUID = 1L; - - private Set assetGuids; - - - public Set getAssetGuids() { - return assetGuids; - } - - public void setAssetGuids(Set assetGuids) { - this.assetGuids = assetGuids; - } - - - @Override - public String toString() { - final StringBuilder sb = new StringBuilder("LinkBusinessPolicyRequest{"); - sb.append("assetGuids=").append(assetGuids); - sb.append('}'); - return sb.toString(); - } -} - diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqueQNAttributeMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqueQNAttributeMigrationService.java index 47e61b2b38..2c8f345791 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqueQNAttributeMigrationService.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqueQNAttributeMigrationService.java @@ -30,6 +30,7 @@ public UniqueQNAttributeMigrationService(EntityGraphRetriever entityRetriever, S public void migrateQN() throws Exception { try { + int count = 0; for (String entityGuid : entityGuids) { AtlasVertex entityVertex = entityRetriever.getEntityVertex(entityGuid); @@ -40,14 +41,20 @@ public void migrateQN() throws Exception { boolean isCommitRequired = migrateuniqueQnAttr(entityVertex); if (isCommitRequired){ - LOG.info("Committing changes for entity: {}", entityGuid); - commitChanges(); + count++; } else { LOG.info("No changes to commit for entity: {} as no migration needed", entityGuid); } } + if (count > 0) { + commitChanges(); + } + else { + LOG.info("No changes to commit for entities as no migration needed"); + } + } catch (Exception e) { LOG.error("Error while migration unique qualifiedName attribute for entities: {}", entityGuids, e); throw e; diff --git a/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java b/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java index a480793cf1..c50e472d84 100644 --- a/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java +++ b/webapp/src/main/java/org/apache/atlas/web/rest/MigrationREST.java @@ -8,7 +8,6 @@ import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.EntityMutationResponse; -import org.apache.atlas.model.instance.UniqueQnMigrationRequest; import org.apache.atlas.repository.graph.GraphHelper; import org.apache.atlas.repository.graphdb.*; import org.apache.atlas.repository.store.graph.AtlasEntityStore; @@ -327,23 +326,23 @@ public List searchForType(@PathParam("typeName") String typeName, @ } @POST - @Path("update-unique-qualified-name") + @Path("repair-unique-qualified-name") @Timed - public Boolean updateUniqueQualifiedName(final UniqueQnMigrationRequest request) throws Exception { + public Boolean updateUniqueQualifiedName(final Set assetGuids) throws Exception { AtlasPerfTracer perf = null; try { - if (CollectionUtils.isEmpty(request.getAssetGuids())) { + if (CollectionUtils.isEmpty(assetGuids)) { throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Asset GUIDs are required for which updating unique qualified name is required"); } if (AtlasPerfTracer.isPerfTraceEnabled(PERF_LOG)) { - perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "MigrationREST.updateUniqueQualifiedName(" + request.getAssetGuids() + ")"); + perf = AtlasPerfTracer.getPerfTracer(PERF_LOG, "MigrationREST.updateUniqueQualifiedName(" + assetGuids + ")"); } - UniqueQNAttributeMigrationService migrationService = new UniqueQNAttributeMigrationService(entityRetriever, request.getAssetGuids(), transactionInterceptHelper); + UniqueQNAttributeMigrationService migrationService = new UniqueQNAttributeMigrationService(entityRetriever, assetGuids, transactionInterceptHelper); migrationService.migrateQN(); } catch (Exception e) { - LOG.error("Error while updating unique qualified name for guids: {}", request.getAssetGuids(), e); + LOG.error("Error while updating unique qualified name for guids: {}", assetGuids, e); throw e; } finally { AtlasPerfTracer.log(perf); From 14234a1126e51319d57a862bfca49cedc4c4ef72 Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Wed, 11 Dec 2024 08:20:24 +0530 Subject: [PATCH 3/4] mesh-294: added info log --- .../store/graph/v2/UniqueQNAttributeMigrationService.java | 1 + 1 file changed, 1 insertion(+) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqueQNAttributeMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqueQNAttributeMigrationService.java index 2c8f345791..52925970ef 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqueQNAttributeMigrationService.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqueQNAttributeMigrationService.java @@ -49,6 +49,7 @@ public void migrateQN() throws Exception { } if (count > 0) { + LOG.info("Total Vertex updated: {}", count); commitChanges(); } else { From 41b2fb863b86a1fffdfe82bb3fa09fc65831cadb Mon Sep 17 00:00:00 2001 From: PRATHAM2002-DS Date: Wed, 11 Dec 2024 11:54:33 +0530 Subject: [PATCH 4/4] mesh-294: added batch in commiting changes --- .../v2/UniqueQNAttributeMigrationService.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqueQNAttributeMigrationService.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqueQNAttributeMigrationService.java index 52925970ef..a7e0fc46e7 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqueQNAttributeMigrationService.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/UniqueQNAttributeMigrationService.java @@ -31,6 +31,7 @@ public UniqueQNAttributeMigrationService(EntityGraphRetriever entityRetriever, S public void migrateQN() throws Exception { try { int count = 0; + int totalUpdatedCount = 0; for (String entityGuid : entityGuids) { AtlasVertex entityVertex = entityRetriever.getEntityVertex(entityGuid); @@ -42,19 +43,25 @@ public void migrateQN() throws Exception { boolean isCommitRequired = migrateuniqueQnAttr(entityVertex); if (isCommitRequired){ count++; + totalUpdatedCount++; } else { LOG.info("No changes to commit for entity: {} as no migration needed", entityGuid); } + + if (count == 20) { + LOG.info("Committing batch of 20 entities..."); + commitChanges(); + count = 0; + } } if (count > 0) { - LOG.info("Total Vertex updated: {}", count); + LOG.info("Committing remaining {} entities...", count); commitChanges(); } - else { - LOG.info("No changes to commit for entities as no migration needed"); - } + + LOG.info("Total Vertex updated: {}", totalUpdatedCount); } catch (Exception e) { LOG.error("Error while migration unique qualifiedName attribute for entities: {}", entityGuids, e);