Skip to content

Commit

Permalink
Merge pull request #3249 from atlanhq/beta-1442-helper
Browse files Browse the repository at this point in the history 
DG-1442 Fixed a minor bug where an empty lexo attribute in update call was removed from vertex
  • Loading branch information
@hr2904
hr2904 authored Jun 14, 2024
2 parents bec0604 + c9857c5 commit f8fed06
Show file tree
Hide file tree
Showing 4 changed files with 97 additions and 133 deletions.
177 changes: 81 additions & 96 deletions .../main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
import org.apache.atlas.util.NanoIdUtils;
import org.apache.atlas.util.lexoRank.LexoRank;
import org.apache.atlas.utils.AtlasEntityUtil;
import org.apache.atlas.v1.model.instance.Id;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
Expand All @@ -27,12 +26,9 @@
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static org.apache.atlas.glossary.GlossaryUtils.ATLAS_GLOSSARY_CATEGORY_TYPENAME;
import static org.apache.atlas.glossary.GlossaryUtils.ATLAS_GLOSSARY_TERM_TYPENAME;
import static org.apache.atlas.repository.Constants.QUERY_COLLECTION_ENTITY_TYPE;
import static org.apache.atlas.repository.Constants.QUALIFIED_NAME;
import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_IS_ENABLED;
import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_SERVICE_NAME;
import static org.apache.atlas.repository.Constants.*;
import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf;
import static org.apache.atlas.type.Constants.LEXICOGRAPHICAL_SORT_ORDER;

Expand Down Expand Up @@ -63,16 +59,12 @@ public class PreProcessorUtils {

public static final String DATA_PRODUCT_EDGE_LABEL = "__DataDomain.dataProducts";
public static final String DOMAIN_PARENT_EDGE_LABEL = "__DataDomain.subDomains";
public static final String STAKEHOLDER_EDGE_LABEL = "__DataDomain.stakeholders";


public static final String PARENT_DOMAIN_QN_ATTR = "parentDomainQualifiedName";
public static final String SUPER_DOMAIN_QN_ATTR = "superDomainQualifiedName";
public static final String DAAP_VISIBILITY_ATTR = "daapVisibility";
public static final String DAAP_VISIBILITY_USERS_ATTR = "daapVisibilityUsers";
public static final String DAAP_VISIBILITY_GROUPS_ATTR = "daapVisibilityGroups";
public static final String OUTPUT_PORT_GUIDS_ATTR = "daapOutputPortGuids";
public static final String INPUT_PORT_GUIDS_ATTR = "daapInputPortGuids";

//Migration Constants
public static final String MIGRATION_TYPE_PREFIX = "MIGRATION:";
Expand All @@ -99,6 +91,13 @@ public enum MigrationStatus {
public static final int REBALANCING_TRIGGER = 119;
public static final int PRE_DELIMITER_LENGTH = 9;
public static final String LEXORANK_HARD_LIMIT = "" + (256 - PRE_DELIMITER_LENGTH);
public static final String LEXORANK_VALID_PATTERN = "^0\\|[0-9a-z]{6}:(?:[0-9a-z]{0," + LEXORANK_HARD_LIMIT + "})?$";
public static final Set<String> ATTRIBUTES;
static {
Set<String> temp = new HashSet<>();
temp.add(LEXICOGRAPHICAL_SORT_ORDER);
ATTRIBUTES = Collections.unmodifiableSet(temp);
}

public static String getUUID(){
return NanoIdUtils.randomNanoId();
Expand Down Expand Up @@ -217,33 +216,45 @@ public static void verifyDuplicateAssetByName(String typeName, String assetName,
}
}

public static void isValidLexoRank(String input, String glossaryQualifiedName, String parentQualifiedName, EntityDiscoveryService discovery) throws AtlasBaseException {
String pattern = "^0\\|[0-9a-z]{6}:(?:[0-9a-z]{0," + LEXORANK_HARD_LIMIT + "})?$";
public static void isValidLexoRank(String entityType, String input, String glossaryQualifiedName, String parentQualifiedName, EntityDiscoveryService discovery) throws AtlasBaseException {
// TODO : To remove this after migration is successful on all tenants and custom-sort is successfully GA
Boolean requestFromMigration = RequestContext.get().getRequestContextHeaders().getOrDefault("x-atlan-request-id", "").contains("custom-sort-migration");
Pattern regex = Pattern.compile(pattern);
Pattern regex = Pattern.compile(LEXORANK_VALID_PATTERN);

Matcher matcher = regex.matcher(input);

if(!matcher.matches()){
throw new AtlasBaseException("Invalid LexicographicSortOrder");
if(!matcher.matches() || StringUtils.isEmpty(input)){
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Invalid value for lexicographicalSortOrder attribute");
}
Boolean requestFromMigration = RequestContext.get().getRequestContextHeaders().getOrDefault("x-atlan-request-id", "").contains("custom-sort-migration");
if(requestFromMigration) {
return;
}
Map<String, String> lexoRankCache = RequestContext.get().getLexoRankCache();
if(Objects.isNull(lexoRankCache)) {
lexoRankCache = new HashMap<>();
}
String cacheKey = entityType + "-" + glossaryQualifiedName + "-" + parentQualifiedName;
if(lexoRankCache.containsKey(cacheKey)){
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Duplicate Lexorank found");
}
Map<String, Object> dslQuery = createDSLforCheckingPreExistingLexoRank(entityType.equals(ATLAS_GLOSSARY_TERM_TYPENAME), input, glossaryQualifiedName, parentQualifiedName);
List<AtlasEntityHeader> assetsWithDuplicateRank = new ArrayList<>();
try {
IndexSearchParams searchParams = new IndexSearchParams();
searchParams.setAttributes(new HashSet<>());
searchParams.setDsl(dslQuery);
assetsWithDuplicateRank = discovery.directIndexSearch(searchParams).getEntities();
} catch (AtlasBaseException e) {
LOG.error("IndexSearch Error Occured : " + e.getMessage());
new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Something went wrong with IndexSearch");
}
if(!requestFromMigration) {
Map<String, Object> dslQuery = createDSLforCheckingPreExistingLexoRank(input, glossaryQualifiedName, parentQualifiedName);
List<AtlasEntityHeader> categories = new ArrayList<>();
try {
IndexSearchParams searchParams = new IndexSearchParams();
searchParams.setAttributes(new HashSet<>());
searchParams.setDsl(dslQuery);
categories = discovery.directIndexSearch(searchParams).getEntities();
} catch (AtlasBaseException e) {
e.printStackTrace();
}

if (!CollectionUtils.isEmpty(categories)) {
throw new AtlasBaseException("Invalid LexicographicSortOrder");
}
if (!CollectionUtils.isEmpty(assetsWithDuplicateRank)) {
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Duplicate Lexorank found");
}

lexoRankCache.put(cacheKey, input);
RequestContext.get().setLexoRankCache(lexoRankCache);
// TODO : Add the rebalancing logic here
int colonIndex = input.indexOf(":");
if (colonIndex != -1 && input.substring(colonIndex + 1).length() >= REBALANCING_TRIGGER) {
Expand All @@ -259,19 +270,18 @@ public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String
}
String lexoRank = "";
String lastLexoRank = "";
boolean isTerm = entity.getTypeName().equals(ATLAS_GLOSSARY_TERM_TYPENAME) ? true : false;

if(lexoRankCache.containsKey(entity.getTypeName() + "-" + glossaryQualifiedName + "-" + parentQualifiedName)) {
lastLexoRank = lexoRankCache.get(entity.getTypeName() + "-" + glossaryQualifiedName + "-" + parentQualifiedName);
boolean isTerm = entity.getTypeName().equals(ATLAS_GLOSSARY_TERM_TYPENAME);
String cacheKey = entity.getTypeName() + "-" + glossaryQualifiedName + "-" + parentQualifiedName;

if(lexoRankCache.containsKey(cacheKey)) {
lastLexoRank = lexoRankCache.get(cacheKey);
} else {
Set<String> attributes = new HashSet<>();
attributes.add(LEXICOGRAPHICAL_SORT_ORDER);

List<AtlasEntityHeader> categories = null;
Map<String, Object> dslQuery = generateDSLQueryForLastCategory(glossaryQualifiedName, parentQualifiedName, isTerm);
Map<String, Object> dslQuery = generateDSLQueryForLastChild(glossaryQualifiedName, parentQualifiedName, isTerm);
try {
IndexSearchParams searchParams = new IndexSearchParams();
searchParams.setAttributes(attributes);
searchParams.setAttributes(ATTRIBUTES);
searchParams.setDsl(dslQuery);
categories = discovery.directIndexSearch(searchParams).getEntities();
} catch (AtlasBaseException e) {
Expand Down Expand Up @@ -301,123 +311,98 @@ public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String
RequestContext.get().setLexoRankCache(lexoRankCache);
}

public static Map<String, Object> createDSLforCheckingPreExistingLexoRank(String lexoRank, String glossaryQualifiedName, String parentQualifiedName) {

Map<String, Object> sortKeyOrder = mapOf(LEXICOGRAPHICAL_SORT_ORDER, mapOf("order", "desc"));
Map<String, Object> scoreSortOrder = mapOf("_score", mapOf("order", "desc"));
Map<String, Object> displayNameSortOrder = mapOf("displayName.keyword", mapOf("order", "desc"));

Object[] sortArray = {sortKeyOrder, scoreSortOrder, displayNameSortOrder};
public static Map<String, Object> createDSLforCheckingPreExistingLexoRank(boolean isTerm, String lexoRank, String glossaryQualifiedName, String parentQualifiedName) {

Map<String, Object> functionScore = mapOf("query", buildBoolQueryDuplicateLexoRank(lexoRank, glossaryQualifiedName, parentQualifiedName));
Map<String, Object> boolMap = buildBoolQueryDuplicateLexoRank(isTerm, lexoRank, glossaryQualifiedName, parentQualifiedName);

Map<String, Object> dsl = new HashMap<>();
dsl.put("from", 0);
dsl.put("size", 100);
dsl.put("sort", sortArray);
dsl.put("query", mapOf("function_score", functionScore));
dsl.put("size", 1);
dsl.put("query", mapOf("bool", boolMap));

return dsl;
}

private static Map<String, Object> buildBoolQueryDuplicateLexoRank(String lexoRank, String glossaryQualifiedName, String parentQualifiedName) {
Map<String, Object> boolQuery = new HashMap<>();
private static Map<String, Object> buildBoolQueryDuplicateLexoRank(boolean isTerm, String lexoRank, String glossaryQualifiedName, String parentQualifiedName) {
Map<String, Object> boolFilter = new HashMap<>();
List<Map<String, Object>> mustArray = new ArrayList<>();
List<Map<String, Object>> mustNotArray = new ArrayList<>();
mustArray.add(mapOf("term", mapOf("__state", "ACTIVE")));
mustArray.add(mapOf("term", mapOf(LEXICOGRAPHICAL_SORT_ORDER, lexoRank)));
if(StringUtils.isNotEmpty(glossaryQualifiedName)) {
mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossaryTerm", "AtlasGlossaryCategory"))));
mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList(ATLAS_GLOSSARY_TERM_TYPENAME, ATLAS_GLOSSARY_CATEGORY_TYPENAME))));
mustArray.add(mapOf("term", mapOf("__glossary", glossaryQualifiedName)));
} else{
mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossary"))));
mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList(ATLAS_GLOSSARY_ENTITY_TYPE))));
}

if(StringUtils.isEmpty(parentQualifiedName)) {
mustNotArray.add(mapOf("exists", mapOf("field", "__categories")));
mustNotArray.add(mapOf("exists", mapOf("field", "__parentCategory")));
List<Map<String, Object>> mustNotArray = new ArrayList<>();
if(isTerm) {
mustNotArray.add(mapOf("exists", mapOf("field", "__categories")));
} else {
mustNotArray.add(mapOf("exists", mapOf("field", "__parentCategory")));
}
boolFilter.put("must_not", mustNotArray);
}
else {
List<Map<String, Object>> shouldParentArray = new ArrayList<>();
shouldParentArray.add(mapOf("term", mapOf("__categories", parentQualifiedName)));
shouldParentArray.add(mapOf("term", mapOf("__parentCategory", parentQualifiedName)));
if(isTerm) {
shouldParentArray.add(mapOf("term", mapOf("__categories", parentQualifiedName)));
} else {
shouldParentArray.add(mapOf("term", mapOf("__parentCategory", parentQualifiedName)));
}
mustArray.add(mapOf("bool",mapOf("should", shouldParentArray)));
}

boolFilter.put("must", mustArray);

Map<String, Object> nestedBoolQuery = mapOf("bool", boolFilter);

Map<String, Object> topBoolFilter = mapOf("filter", nestedBoolQuery);

boolQuery.put("bool", topBoolFilter);

return boolQuery;
return boolFilter;
}
public static Map<String, Object> generateDSLQueryForLastCategory(String glossaryQualifiedName, String parentQualifiedName, boolean isTerm) {
public static Map<String, Object> generateDSLQueryForLastChild(String glossaryQualifiedName, String parentQualifiedName, boolean isTerm) {

Map<String, Object> sortKeyOrder = mapOf(LEXICOGRAPHICAL_SORT_ORDER, mapOf("order", "desc"));
Map<String, Object> scoreSortOrder = mapOf("_score", mapOf("order", "desc"));
Map<String, Object> displayNameSortOrder = mapOf("displayName.keyword", mapOf("order", "desc"));

Object[] sortArray = {sortKeyOrder, scoreSortOrder, displayNameSortOrder};
Object[] sortArray = {sortKeyOrder};

Map<String, Object> functionScore = mapOf("query", buildBoolQuery(glossaryQualifiedName, parentQualifiedName, isTerm));
Map<String, Object> boolMap = buildBoolQuery(glossaryQualifiedName, parentQualifiedName, isTerm);

Map<String, Object> dsl = new HashMap<>();
dsl.put("from", 0);
dsl.put("size", 1);
dsl.put("sort", sortArray);
dsl.put("query", mapOf("function_score", functionScore));
dsl.put("query", mapOf("bool", boolMap));

return dsl;
}

private static Map<String, Object> buildBoolQuery(String glossaryQualifiedName, String parentQualifiedName, boolean isTerm) {
Map<String, Object> boolQuery = new HashMap<>();
int mustArrayLength = 0;
if(StringUtils.isEmpty(parentQualifiedName) && StringUtils.isEmpty(glossaryQualifiedName)){
mustArrayLength = 2;
} else if(StringUtils.isEmpty(parentQualifiedName) && StringUtils.isNotEmpty(glossaryQualifiedName)){
mustArrayLength = 3;
} else {
mustArrayLength = 4;
}
Map<String, Object>[] mustArray = new Map[mustArrayLength];
Map<String, Object> boolFilter = new HashMap<>();
Map<String, Object>[] mustNotArray = new Map[2];
List<Map<String, Object>> mustArray = new ArrayList<>();
List<Map<String, Object>> mustNotArray = new ArrayList<>();

mustArray[0] = mapOf("term", mapOf("__state", "ACTIVE"));
mustArray.add(mapOf("term", mapOf("__state", "ACTIVE")));
if(StringUtils.isNotEmpty(glossaryQualifiedName)) {
String typeName = isTerm ? "AtlasGlossaryTerm" : "AtlasGlossaryCategory";
mustArray[1] = mapOf("term", mapOf("__typeName.keyword", typeName));
mustArray[2] = mapOf("term", mapOf("__glossary", glossaryQualifiedName));
mustArray.add(mapOf("term", mapOf("__typeName.keyword", typeName)));
mustArray.add(mapOf("term", mapOf("__glossary", glossaryQualifiedName)));
} else{
mustArray[1] = mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossary")));
mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossary"))));
}

if(StringUtils.isEmpty(parentQualifiedName)) {
mustNotArray[0] = mapOf("exists", mapOf("field", "__categories"));
mustNotArray[1] = mapOf("exists", mapOf("field", "__parentCategory"));
mustNotArray.add(mapOf("exists", mapOf("field", "__categories")));
mustNotArray.add(mapOf("exists", mapOf("field", "__parentCategory")));
boolFilter.put("must_not", mustNotArray);
}
else {
Map<String, Object>[] shouldParentArray = new Map[2];
shouldParentArray[0] = mapOf("term", mapOf("__categories", parentQualifiedName));
shouldParentArray[1] = mapOf("term", mapOf("__parentCategory", parentQualifiedName));
mustArray[3] = mapOf("bool",mapOf("should", shouldParentArray));
mustArray.add(mapOf("bool",mapOf("should", shouldParentArray)));
}

boolFilter.put("must", mustArray);

Map<String, Object> nestedBoolQuery = mapOf("bool", boolFilter);

Map<String, Object> topBoolFilter = mapOf("filter", nestedBoolQuery);

boolQuery.put("bool", topBoolFilter);

return boolQuery;
return boolFilter;
}
}
18 changes: 5 additions & 13 deletions ...rg/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,6 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Objects;
import java.util.stream.Collectors;

import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST;
Expand Down Expand Up @@ -135,7 +134,7 @@ private void processCreateCategory(AtlasEntity entity, AtlasVertex vertex) throw
if(StringUtils.isEmpty(lexicographicalSortOrder)){
assignNewLexicographicalSortOrder(entity,glossaryQualifiedName, parentQname, this.discovery);
} else {
isValidLexoRank(lexicographicalSortOrder, glossaryQualifiedName, parentQname, this.discovery);
isValidLexoRank(entity.getTypeName(), lexicographicalSortOrder, glossaryQualifiedName, parentQname, this.discovery);
}

entity.setAttribute(QUALIFIED_NAME, createQualifiedName(vertex));
Expand Down Expand Up @@ -169,7 +168,10 @@ private void processUpdateCategory(AtlasEntity entity, AtlasVertex vertex) throw
parentQname = (String) parentCategory.getAttribute(QUALIFIED_NAME);
}
if(StringUtils.isNotEmpty(lexicographicalSortOrder)) {
isValidLexoRank(lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery);
isValidLexoRank(entity.getTypeName(), lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery);
} else {
lexicographicalSortOrder = (String) storedCategory.getAttribute(LEXICOGRAPHICAL_SORT_ORDER);
entity.setAttribute(LEXICOGRAPHICAL_SORT_ORDER, lexicographicalSortOrder);
}

if (!currentGlossaryQualifiedName.equals(newGlossaryQualifiedName)){
Expand Down Expand Up @@ -294,7 +296,6 @@ public void moveChildTermToAnotherGlossary(AtlasVertex termVertex,

//check duplicate term name
termExists(termName, targetGlossaryQualifiedName);
ensureOnlyOneCategoryIsAssociated(termVertex);

String currentTermQualifiedName = termVertex.getProperty(QUALIFIED_NAME, String.class);
String updatedTermQualifiedName = currentTermQualifiedName.replace(sourceGlossaryQualifiedName, targetGlossaryQualifiedName);
Expand Down Expand Up @@ -331,15 +332,6 @@ public void moveChildTermToAnotherGlossary(AtlasVertex termVertex,
}
}

private void ensureOnlyOneCategoryIsAssociated(AtlasVertex vertex) throws AtlasBaseException {
final Integer numOfCategoryEdges = GraphHelper.getCountOfCategoryEdges(vertex);

if(numOfCategoryEdges>1) {
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Cannot move term with multiple " +
"categories associated to another glossary");
}
}

private void validateParentForGlossaryChange(AtlasEntity category,
AtlasVertex categoryVertex,
String targetGlossaryQualifiedName) throws AtlasBaseException {
Expand Down
Loading

0 comments on commit f8fed06

Please sign in to comment.