Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Revert "DG-1442 Fixed a minor bug where an empty lexo attribute in update call was removed from vertex" #3251

Merged
merged 1 commit into from
Jun 14, 2024
Merged

Revert "DG-1442 Fixed a minor bug where an empty lexo attribute in update call was removed from vertex" #3251

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
177 changes: 96 additions & 81 deletions .../main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import org.apache.atlas.util.NanoIdUtils;
import org.apache.atlas.util.lexoRank.LexoRank;
import org.apache.atlas.utils.AtlasEntityUtil;
import org.apache.atlas.v1.model.instance.Id;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
Expand All @@ -26,9 +27,12 @@
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static org.apache.atlas.glossary.GlossaryUtils.ATLAS_GLOSSARY_CATEGORY_TYPENAME;
import static org.apache.atlas.glossary.GlossaryUtils.ATLAS_GLOSSARY_TERM_TYPENAME;
import static org.apache.atlas.repository.Constants.*;
import static org.apache.atlas.repository.Constants.QUERY_COLLECTION_ENTITY_TYPE;
import static org.apache.atlas.repository.Constants.QUALIFIED_NAME;
import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_IS_ENABLED;
import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_SERVICE_NAME;
import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf;
import static org.apache.atlas.type.Constants.LEXICOGRAPHICAL_SORT_ORDER;

Expand Down Expand Up @@ -59,12 +63,16 @@ public class PreProcessorUtils {

public static final String DATA_PRODUCT_EDGE_LABEL = "__DataDomain.dataProducts";
public static final String DOMAIN_PARENT_EDGE_LABEL = "__DataDomain.subDomains";
public static final String STAKEHOLDER_EDGE_LABEL = "__DataDomain.stakeholders";


public static final String PARENT_DOMAIN_QN_ATTR = "parentDomainQualifiedName";
public static final String SUPER_DOMAIN_QN_ATTR = "superDomainQualifiedName";
public static final String DAAP_VISIBILITY_ATTR = "daapVisibility";
public static final String DAAP_VISIBILITY_USERS_ATTR = "daapVisibilityUsers";
public static final String DAAP_VISIBILITY_GROUPS_ATTR = "daapVisibilityGroups";
public static final String OUTPUT_PORT_GUIDS_ATTR = "daapOutputPortGuids";
public static final String INPUT_PORT_GUIDS_ATTR = "daapInputPortGuids";

//Migration Constants
public static final String MIGRATION_TYPE_PREFIX = "MIGRATION:";
Expand All @@ -91,13 +99,6 @@ public enum MigrationStatus {
public static final int REBALANCING_TRIGGER = 119;
public static final int PRE_DELIMITER_LENGTH = 9;
public static final String LEXORANK_HARD_LIMIT = "" + (256 - PRE_DELIMITER_LENGTH);
public static final String LEXORANK_VALID_PATTERN = "^0\\|[0-9a-z]{6}:(?:[0-9a-z]{0," + LEXORANK_HARD_LIMIT + "})?$";
public static final Set<String> ATTRIBUTES;
static {
Set<String> temp = new HashSet<>();
temp.add(LEXICOGRAPHICAL_SORT_ORDER);
ATTRIBUTES = Collections.unmodifiableSet(temp);
}

public static String getUUID(){
return NanoIdUtils.randomNanoId();
Expand Down Expand Up @@ -216,45 +217,33 @@ public static void verifyDuplicateAssetByName(String typeName, String assetName,
}
}

public static void isValidLexoRank(String entityType, String input, String glossaryQualifiedName, String parentQualifiedName, EntityDiscoveryService discovery) throws AtlasBaseException {
public static void isValidLexoRank(String input, String glossaryQualifiedName, String parentQualifiedName, EntityDiscoveryService discovery) throws AtlasBaseException {
String pattern = "^0\\|[0-9a-z]{6}:(?:[0-9a-z]{0," + LEXORANK_HARD_LIMIT + "})?$";
// TODO : To remove this after migration is successful on all tenants and custom-sort is successfully GA
Pattern regex = Pattern.compile(LEXORANK_VALID_PATTERN);
Boolean requestFromMigration = RequestContext.get().getRequestContextHeaders().getOrDefault("x-atlan-request-id", "").contains("custom-sort-migration");
Pattern regex = Pattern.compile(pattern);

Matcher matcher = regex.matcher(input);

if(!matcher.matches() || StringUtils.isEmpty(input)){
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Invalid value for lexicographicalSortOrder attribute");
}
Boolean requestFromMigration = RequestContext.get().getRequestContextHeaders().getOrDefault("x-atlan-request-id", "").contains("custom-sort-migration");
if(requestFromMigration) {
return;
}
Map<String, String> lexoRankCache = RequestContext.get().getLexoRankCache();
if(Objects.isNull(lexoRankCache)) {
lexoRankCache = new HashMap<>();
}
String cacheKey = entityType + "-" + glossaryQualifiedName + "-" + parentQualifiedName;
if(lexoRankCache.containsKey(cacheKey)){
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Duplicate Lexorank found");
}
Map<String, Object> dslQuery = createDSLforCheckingPreExistingLexoRank(entityType.equals(ATLAS_GLOSSARY_TERM_TYPENAME), input, glossaryQualifiedName, parentQualifiedName);
List<AtlasEntityHeader> assetsWithDuplicateRank = new ArrayList<>();
try {
IndexSearchParams searchParams = new IndexSearchParams();
searchParams.setAttributes(new HashSet<>());
searchParams.setDsl(dslQuery);
assetsWithDuplicateRank = discovery.directIndexSearch(searchParams).getEntities();
} catch (AtlasBaseException e) {
LOG.error("IndexSearch Error Occured : " + e.getMessage());
new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Something went wrong with IndexSearch");
if(!matcher.matches()){
throw new AtlasBaseException("Invalid LexicographicSortOrder");
}
if(!requestFromMigration) {
Map<String, Object> dslQuery = createDSLforCheckingPreExistingLexoRank(input, glossaryQualifiedName, parentQualifiedName);
List<AtlasEntityHeader> categories = new ArrayList<>();
try {
IndexSearchParams searchParams = new IndexSearchParams();
searchParams.setAttributes(new HashSet<>());
searchParams.setDsl(dslQuery);
categories = discovery.directIndexSearch(searchParams).getEntities();
} catch (AtlasBaseException e) {
e.printStackTrace();
}

if (!CollectionUtils.isEmpty(assetsWithDuplicateRank)) {
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Duplicate Lexorank found");
if (!CollectionUtils.isEmpty(categories)) {
throw new AtlasBaseException("Invalid LexicographicSortOrder");
}
}

lexoRankCache.put(cacheKey, input);
RequestContext.get().setLexoRankCache(lexoRankCache);
// TODO : Add the rebalancing logic here
int colonIndex = input.indexOf(":");
if (colonIndex != -1 && input.substring(colonIndex + 1).length() >= REBALANCING_TRIGGER) {
Expand All @@ -270,18 +259,19 @@ public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String
}
String lexoRank = "";
String lastLexoRank = "";
boolean isTerm = entity.getTypeName().equals(ATLAS_GLOSSARY_TERM_TYPENAME);
String cacheKey = entity.getTypeName() + "-" + glossaryQualifiedName + "-" + parentQualifiedName;
boolean isTerm = entity.getTypeName().equals(ATLAS_GLOSSARY_TERM_TYPENAME) ? true : false;

if(lexoRankCache.containsKey(cacheKey)) {
lastLexoRank = lexoRankCache.get(cacheKey);
} else {
if(lexoRankCache.containsKey(entity.getTypeName() + "-" + glossaryQualifiedName + "-" + parentQualifiedName)) {
lastLexoRank = lexoRankCache.get(entity.getTypeName() + "-" + glossaryQualifiedName + "-" + parentQualifiedName);

} else {
Set<String> attributes = new HashSet<>();
attributes.add(LEXICOGRAPHICAL_SORT_ORDER);
List<AtlasEntityHeader> categories = null;
Map<String, Object> dslQuery = generateDSLQueryForLastChild(glossaryQualifiedName, parentQualifiedName, isTerm);
Map<String, Object> dslQuery = generateDSLQueryForLastCategory(glossaryQualifiedName, parentQualifiedName, isTerm);
try {
IndexSearchParams searchParams = new IndexSearchParams();
searchParams.setAttributes(ATTRIBUTES);
searchParams.setAttributes(attributes);
searchParams.setDsl(dslQuery);
categories = discovery.directIndexSearch(searchParams).getEntities();
} catch (AtlasBaseException e) {
Expand Down Expand Up @@ -311,98 +301,123 @@ public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String
RequestContext.get().setLexoRankCache(lexoRankCache);
}

public static Map<String, Object> createDSLforCheckingPreExistingLexoRank(boolean isTerm, String lexoRank, String glossaryQualifiedName, String parentQualifiedName) {
public static Map<String, Object> createDSLforCheckingPreExistingLexoRank(String lexoRank, String glossaryQualifiedName, String parentQualifiedName) {

Map<String, Object> sortKeyOrder = mapOf(LEXICOGRAPHICAL_SORT_ORDER, mapOf("order", "desc"));
Map<String, Object> scoreSortOrder = mapOf("_score", mapOf("order", "desc"));
Map<String, Object> displayNameSortOrder = mapOf("displayName.keyword", mapOf("order", "desc"));

Object[] sortArray = {sortKeyOrder, scoreSortOrder, displayNameSortOrder};

Map<String, Object> boolMap = buildBoolQueryDuplicateLexoRank(isTerm, lexoRank, glossaryQualifiedName, parentQualifiedName);
Map<String, Object> functionScore = mapOf("query", buildBoolQueryDuplicateLexoRank(lexoRank, glossaryQualifiedName, parentQualifiedName));

Map<String, Object> dsl = new HashMap<>();
dsl.put("from", 0);
dsl.put("size", 1);
dsl.put("query", mapOf("bool", boolMap));
dsl.put("size", 100);
dsl.put("sort", sortArray);
dsl.put("query", mapOf("function_score", functionScore));

return dsl;
}

private static Map<String, Object> buildBoolQueryDuplicateLexoRank(boolean isTerm, String lexoRank, String glossaryQualifiedName, String parentQualifiedName) {
private static Map<String, Object> buildBoolQueryDuplicateLexoRank(String lexoRank, String glossaryQualifiedName, String parentQualifiedName) {
Map<String, Object> boolQuery = new HashMap<>();
Map<String, Object> boolFilter = new HashMap<>();
List<Map<String, Object>> mustArray = new ArrayList<>();
List<Map<String, Object>> mustNotArray = new ArrayList<>();
mustArray.add(mapOf("term", mapOf("__state", "ACTIVE")));
mustArray.add(mapOf("term", mapOf(LEXICOGRAPHICAL_SORT_ORDER, lexoRank)));
if(StringUtils.isNotEmpty(glossaryQualifiedName)) {
mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList(ATLAS_GLOSSARY_TERM_TYPENAME, ATLAS_GLOSSARY_CATEGORY_TYPENAME))));
mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossaryTerm", "AtlasGlossaryCategory"))));
mustArray.add(mapOf("term", mapOf("__glossary", glossaryQualifiedName)));
} else{
mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList(ATLAS_GLOSSARY_ENTITY_TYPE))));
mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossary"))));
}

if(StringUtils.isEmpty(parentQualifiedName)) {
List<Map<String, Object>> mustNotArray = new ArrayList<>();
if(isTerm) {
mustNotArray.add(mapOf("exists", mapOf("field", "__categories")));
} else {
mustNotArray.add(mapOf("exists", mapOf("field", "__parentCategory")));
}
mustNotArray.add(mapOf("exists", mapOf("field", "__categories")));
mustNotArray.add(mapOf("exists", mapOf("field", "__parentCategory")));
boolFilter.put("must_not", mustNotArray);
}
else {
List<Map<String, Object>> shouldParentArray = new ArrayList<>();
if(isTerm) {
shouldParentArray.add(mapOf("term", mapOf("__categories", parentQualifiedName)));
} else {
shouldParentArray.add(mapOf("term", mapOf("__parentCategory", parentQualifiedName)));
}
shouldParentArray.add(mapOf("term", mapOf("__categories", parentQualifiedName)));
shouldParentArray.add(mapOf("term", mapOf("__parentCategory", parentQualifiedName)));
mustArray.add(mapOf("bool",mapOf("should", shouldParentArray)));
}

boolFilter.put("must", mustArray);

return boolFilter;
Map<String, Object> nestedBoolQuery = mapOf("bool", boolFilter);

Map<String, Object> topBoolFilter = mapOf("filter", nestedBoolQuery);

boolQuery.put("bool", topBoolFilter);

return boolQuery;
}
public static Map<String, Object> generateDSLQueryForLastChild(String glossaryQualifiedName, String parentQualifiedName, boolean isTerm) {
public static Map<String, Object> generateDSLQueryForLastCategory(String glossaryQualifiedName, String parentQualifiedName, boolean isTerm) {

Map<String, Object> sortKeyOrder = mapOf(LEXICOGRAPHICAL_SORT_ORDER, mapOf("order", "desc"));
Map<String, Object> scoreSortOrder = mapOf("_score", mapOf("order", "desc"));
Map<String, Object> displayNameSortOrder = mapOf("displayName.keyword", mapOf("order", "desc"));

Object[] sortArray = {sortKeyOrder};
Object[] sortArray = {sortKeyOrder, scoreSortOrder, displayNameSortOrder};

Map<String, Object> boolMap = buildBoolQuery(glossaryQualifiedName, parentQualifiedName, isTerm);
Map<String, Object> functionScore = mapOf("query", buildBoolQuery(glossaryQualifiedName, parentQualifiedName, isTerm));

Map<String, Object> dsl = new HashMap<>();
dsl.put("from", 0);
dsl.put("size", 1);
dsl.put("sort", sortArray);
dsl.put("query", mapOf("bool", boolMap));
dsl.put("query", mapOf("function_score", functionScore));

return dsl;
}

private static Map<String, Object> buildBoolQuery(String glossaryQualifiedName, String parentQualifiedName, boolean isTerm) {
Map<String, Object> boolQuery = new HashMap<>();
int mustArrayLength = 0;
if(StringUtils.isEmpty(parentQualifiedName) && StringUtils.isEmpty(glossaryQualifiedName)){
mustArrayLength = 2;
} else if(StringUtils.isEmpty(parentQualifiedName) && StringUtils.isNotEmpty(glossaryQualifiedName)){
mustArrayLength = 3;
} else {
mustArrayLength = 4;
}
Map<String, Object>[] mustArray = new Map[mustArrayLength];
Map<String, Object> boolFilter = new HashMap<>();
List<Map<String, Object>> mustArray = new ArrayList<>();
List<Map<String, Object>> mustNotArray = new ArrayList<>();
Map<String, Object>[] mustNotArray = new Map[2];

mustArray.add(mapOf("term", mapOf("__state", "ACTIVE")));
mustArray[0] = mapOf("term", mapOf("__state", "ACTIVE"));
if(StringUtils.isNotEmpty(glossaryQualifiedName)) {
String typeName = isTerm ? "AtlasGlossaryTerm" : "AtlasGlossaryCategory";
mustArray.add(mapOf("term", mapOf("__typeName.keyword", typeName)));
mustArray.add(mapOf("term", mapOf("__glossary", glossaryQualifiedName)));
mustArray[1] = mapOf("term", mapOf("__typeName.keyword", typeName));
mustArray[2] = mapOf("term", mapOf("__glossary", glossaryQualifiedName));
} else{
mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossary"))));
mustArray[1] = mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossary")));
}

if(StringUtils.isEmpty(parentQualifiedName)) {
mustNotArray.add(mapOf("exists", mapOf("field", "__categories")));
mustNotArray.add(mapOf("exists", mapOf("field", "__parentCategory")));
mustNotArray[0] = mapOf("exists", mapOf("field", "__categories"));
mustNotArray[1] = mapOf("exists", mapOf("field", "__parentCategory"));
boolFilter.put("must_not", mustNotArray);
}
else {
Map<String, Object>[] shouldParentArray = new Map[2];
shouldParentArray[0] = mapOf("term", mapOf("__categories", parentQualifiedName));
shouldParentArray[1] = mapOf("term", mapOf("__parentCategory", parentQualifiedName));
mustArray.add(mapOf("bool",mapOf("should", shouldParentArray)));
mustArray[3] = mapOf("bool",mapOf("should", shouldParentArray));
}

boolFilter.put("must", mustArray);

return boolFilter;
Map<String, Object> nestedBoolQuery = mapOf("bool", boolFilter);

Map<String, Object> topBoolFilter = mapOf("filter", nestedBoolQuery);

boolQuery.put("bool", topBoolFilter);

return boolQuery;
}
}
18 changes: 13 additions & 5 deletions ...rg/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.Objects;
import java.util.stream.Collectors;

import static org.apache.atlas.AtlasErrorCode.BAD_REQUEST;
Expand Down Expand Up @@ -134,7 +135,7 @@ private void processCreateCategory(AtlasEntity entity, AtlasVertex vertex) throw
if(StringUtils.isEmpty(lexicographicalSortOrder)){
assignNewLexicographicalSortOrder(entity,glossaryQualifiedName, parentQname, this.discovery);
} else {
isValidLexoRank(entity.getTypeName(), lexicographicalSortOrder, glossaryQualifiedName, parentQname, this.discovery);
isValidLexoRank(lexicographicalSortOrder, glossaryQualifiedName, parentQname, this.discovery);
}

entity.setAttribute(QUALIFIED_NAME, createQualifiedName(vertex));
Expand Down Expand Up @@ -168,10 +169,7 @@ private void processUpdateCategory(AtlasEntity entity, AtlasVertex vertex) throw
parentQname = (String) parentCategory.getAttribute(QUALIFIED_NAME);
}
if(StringUtils.isNotEmpty(lexicographicalSortOrder)) {
isValidLexoRank(entity.getTypeName(), lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery);
} else {
lexicographicalSortOrder = (String) storedCategory.getAttribute(LEXICOGRAPHICAL_SORT_ORDER);
entity.setAttribute(LEXICOGRAPHICAL_SORT_ORDER, lexicographicalSortOrder);
isValidLexoRank(lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery);
}

if (!currentGlossaryQualifiedName.equals(newGlossaryQualifiedName)){
Expand Down Expand Up @@ -296,6 +294,7 @@ public void moveChildTermToAnotherGlossary(AtlasVertex termVertex,

//check duplicate term name
termExists(termName, targetGlossaryQualifiedName);
ensureOnlyOneCategoryIsAssociated(termVertex);

String currentTermQualifiedName = termVertex.getProperty(QUALIFIED_NAME, String.class);
String updatedTermQualifiedName = currentTermQualifiedName.replace(sourceGlossaryQualifiedName, targetGlossaryQualifiedName);
Expand Down Expand Up @@ -332,6 +331,15 @@ public void moveChildTermToAnotherGlossary(AtlasVertex termVertex,
}
}

private void ensureOnlyOneCategoryIsAssociated(AtlasVertex vertex) throws AtlasBaseException {
final Integer numOfCategoryEdges = GraphHelper.getCountOfCategoryEdges(vertex);

if(numOfCategoryEdges>1) {
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Cannot move term with multiple " +
"categories associated to another glossary");
}
}

private void validateParentForGlossaryChange(AtlasEntity category,
AtlasVertex categoryVertex,
String targetGlossaryQualifiedName) throws AtlasBaseException {
Expand Down
Loading
Loading