Skip to content

Commit

Permalink
Fixed PR comments
Browse files Browse the repository at this point in the history
  • Loading branch information
@hr2904
hr2904 committed Jun 12, 2024
1 parent 8279cce commit d5959d1
Show file tree
Hide file tree
Showing 5 changed files with 76 additions and 129 deletions.
161 changes: 70 additions & 91 deletions .../main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
import org.apache.atlas.util.NanoIdUtils;
import org.apache.atlas.util.lexoRank.LexoRank;
import org.apache.atlas.utils.AtlasEntityUtil;
import org.apache.atlas.v1.model.instance.Id;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
Expand All @@ -27,12 +26,9 @@
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import static org.apache.atlas.glossary.GlossaryUtils.ATLAS_GLOSSARY_CATEGORY_TYPENAME;
import static org.apache.atlas.glossary.GlossaryUtils.ATLAS_GLOSSARY_TERM_TYPENAME;
import static org.apache.atlas.repository.Constants.QUERY_COLLECTION_ENTITY_TYPE;
import static org.apache.atlas.repository.Constants.QUALIFIED_NAME;
import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY;
import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_IS_ENABLED;
import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_SERVICE_NAME;
import static org.apache.atlas.repository.Constants.*;
import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf;
import static org.apache.atlas.type.Constants.LEXICOGRAPHICAL_SORT_ORDER;

Expand Down Expand Up @@ -95,6 +91,13 @@ public enum MigrationStatus {
public static final int REBALANCING_TRIGGER = 119;
public static final int PRE_DELIMITER_LENGTH = 9;
public static final String LEXORANK_HARD_LIMIT = "" + (256 - PRE_DELIMITER_LENGTH);
public static final String LEXORANK_VALID_PATTERN = "^0\\|[0-9a-z]{6}:(?:[0-9a-z]{0," + LEXORANK_HARD_LIMIT + "})?$";
public static final Set<String> ATTRIBUTES;
static {
Set<String> temp = new HashSet<>();
temp.add(LEXICOGRAPHICAL_SORT_ORDER);
ATTRIBUTES = Collections.unmodifiableSet(temp);
}

public static String getUUID(){
return NanoIdUtils.randomNanoId();
Expand Down Expand Up @@ -213,33 +216,35 @@ public static void verifyDuplicateAssetByName(String typeName, String assetName,
}
}

public static void isValidLexoRank(String input, String glossaryQualifiedName, String parentQualifiedName, EntityDiscoveryService discovery) throws AtlasBaseException {
String pattern = "^0\\|[0-9a-z]{6}:(?:[0-9a-z]{0," + LEXORANK_HARD_LIMIT + "})?$";
public static void isValidLexoRank(boolean isTerm, String input, String glossaryQualifiedName, String parentQualifiedName, EntityDiscoveryService discovery) throws AtlasBaseException {
// TODO : To remove this after migration is successful on all tenants and custom-sort is successfully GA
Boolean requestFromMigration = RequestContext.get().getRequestContextHeaders().getOrDefault("x-atlan-request-id", "").contains("custom-sort-migration");
Pattern regex = Pattern.compile(pattern);
Pattern regex = Pattern.compile(LEXORANK_VALID_PATTERN);

Matcher matcher = regex.matcher(input);

if(!matcher.matches()){
throw new AtlasBaseException("Invalid LexicographicSortOrder");
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Invalid value for attribute");
}
Boolean requestFromMigration = RequestContext.get().getRequestContextHeaders().getOrDefault("x-atlan-request-id", "").contains("custom-sort-migration");
if(requestFromMigration) {
return;
}
Map<String, Object> dslQuery = createDSLforCheckingPreExistingLexoRank(isTerm, input, glossaryQualifiedName, parentQualifiedName);
List<AtlasEntityHeader> assetsWithDuplicateRank = new ArrayList<>();
try {
IndexSearchParams searchParams = new IndexSearchParams();
searchParams.setAttributes(new HashSet<>());
searchParams.setDsl(dslQuery);
assetsWithDuplicateRank = discovery.directIndexSearch(searchParams).getEntities();
} catch (AtlasBaseException e) {
LOG.error("IndexSearch Error Occured : " + e.getMessage());
new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Something went wrong with IndexSearch");
}
if(!requestFromMigration) {
Map<String, Object> dslQuery = createDSLforCheckingPreExistingLexoRank(input, glossaryQualifiedName, parentQualifiedName);
List<AtlasEntityHeader> categories = new ArrayList<>();
try {
IndexSearchParams searchParams = new IndexSearchParams();
searchParams.setAttributes(new HashSet<>());
searchParams.setDsl(dslQuery);
categories = discovery.directIndexSearch(searchParams).getEntities();
} catch (AtlasBaseException e) {
e.printStackTrace();
}

if (!CollectionUtils.isEmpty(categories)) {
throw new AtlasBaseException("Invalid LexicographicSortOrder");
}
if (!CollectionUtils.isEmpty(assetsWithDuplicateRank)) {
throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Duplicate Lexorank found");
}

// TODO : Add the rebalancing logic here
int colonIndex = input.indexOf(":");
if (colonIndex != -1 && input.substring(colonIndex + 1).length() >= REBALANCING_TRIGGER) {
Expand All @@ -255,19 +260,18 @@ public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String
}
String lexoRank = "";
String lastLexoRank = "";
boolean isTerm = entity.getTypeName().equals(ATLAS_GLOSSARY_TERM_TYPENAME) ? true : false;

if(lexoRankCache.containsKey(entity.getTypeName() + "-" + glossaryQualifiedName + "-" + parentQualifiedName)) {
lastLexoRank = lexoRankCache.get(entity.getTypeName() + "-" + glossaryQualifiedName + "-" + parentQualifiedName);
boolean isTerm = entity.getTypeName().equals(ATLAS_GLOSSARY_TERM_TYPENAME);
String cacheKey = entity.getTypeName() + "-" + glossaryQualifiedName + "-" + parentQualifiedName;

if(lexoRankCache.containsKey(cacheKey)) {
lastLexoRank = lexoRankCache.get(cacheKey);
} else {
Set<String> attributes = new HashSet<>();
attributes.add(LEXICOGRAPHICAL_SORT_ORDER);

List<AtlasEntityHeader> categories = null;
Map<String, Object> dslQuery = generateDSLQueryForLastCategory(glossaryQualifiedName, parentQualifiedName, isTerm);
Map<String, Object> dslQuery = generateDSLQueryForLastChild(glossaryQualifiedName, parentQualifiedName, isTerm);
try {
IndexSearchParams searchParams = new IndexSearchParams();
searchParams.setAttributes(attributes);
searchParams.setAttributes(ATTRIBUTES);
searchParams.setDsl(dslQuery);
categories = discovery.directIndexSearch(searchParams).getEntities();
} catch (AtlasBaseException e) {
Expand Down Expand Up @@ -297,123 +301,98 @@ public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String
RequestContext.get().setLexoRankCache(lexoRankCache);
}

public static Map<String, Object> createDSLforCheckingPreExistingLexoRank(String lexoRank, String glossaryQualifiedName, String parentQualifiedName) {

Map<String, Object> sortKeyOrder = mapOf(LEXICOGRAPHICAL_SORT_ORDER, mapOf("order", "desc"));
Map<String, Object> scoreSortOrder = mapOf("_score", mapOf("order", "desc"));
Map<String, Object> displayNameSortOrder = mapOf("displayName.keyword", mapOf("order", "desc"));
public static Map<String, Object> createDSLforCheckingPreExistingLexoRank(boolean isTerm, String lexoRank, String glossaryQualifiedName, String parentQualifiedName) {

Object[] sortArray = {sortKeyOrder, scoreSortOrder, displayNameSortOrder};

Map<String, Object> functionScore = mapOf("query", buildBoolQueryDuplicateLexoRank(lexoRank, glossaryQualifiedName, parentQualifiedName));
Map<String, Object> boolMap = buildBoolQueryDuplicateLexoRank(isTerm, lexoRank, glossaryQualifiedName, parentQualifiedName);

Map<String, Object> dsl = new HashMap<>();
dsl.put("from", 0);
dsl.put("size", 100);
dsl.put("sort", sortArray);
dsl.put("query", mapOf("function_score", functionScore));
dsl.put("size", 1);
dsl.put("query", mapOf("bool", boolMap));

return dsl;
}

private static Map<String, Object> buildBoolQueryDuplicateLexoRank(String lexoRank, String glossaryQualifiedName, String parentQualifiedName) {
Map<String, Object> boolQuery = new HashMap<>();
private static Map<String, Object> buildBoolQueryDuplicateLexoRank(boolean isTerm, String lexoRank, String glossaryQualifiedName, String parentQualifiedName) {
Map<String, Object> boolFilter = new HashMap<>();
List<Map<String, Object>> mustArray = new ArrayList<>();
List<Map<String, Object>> mustNotArray = new ArrayList<>();
mustArray.add(mapOf("term", mapOf("__state", "ACTIVE")));
mustArray.add(mapOf("term", mapOf(LEXICOGRAPHICAL_SORT_ORDER, lexoRank)));
if(StringUtils.isNotEmpty(glossaryQualifiedName)) {
mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossaryTerm", "AtlasGlossaryCategory"))));
mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList(ATLAS_GLOSSARY_TERM_TYPENAME, ATLAS_GLOSSARY_CATEGORY_TYPENAME))));
mustArray.add(mapOf("term", mapOf("__glossary", glossaryQualifiedName)));
} else{
mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossary"))));
mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList(ATLAS_GLOSSARY_ENTITY_TYPE))));
}

if(StringUtils.isEmpty(parentQualifiedName)) {
mustNotArray.add(mapOf("exists", mapOf("field", "__categories")));
mustNotArray.add(mapOf("exists", mapOf("field", "__parentCategory")));
List<Map<String, Object>> mustNotArray = new ArrayList<>();
if(isTerm) {
mustNotArray.add(mapOf("exists", mapOf("field", "__categories")));
} else {
mustNotArray.add(mapOf("exists", mapOf("field", "__parentCategory")));
}
boolFilter.put("must_not", mustNotArray);
}
else {
List<Map<String, Object>> shouldParentArray = new ArrayList<>();
shouldParentArray.add(mapOf("term", mapOf("__categories", parentQualifiedName)));
shouldParentArray.add(mapOf("term", mapOf("__parentCategory", parentQualifiedName)));
if(isTerm) {
shouldParentArray.add(mapOf("term", mapOf("__categories", parentQualifiedName)));
} else {
shouldParentArray.add(mapOf("term", mapOf("__parentCategory", parentQualifiedName)));
}
mustArray.add(mapOf("bool",mapOf("should", shouldParentArray)));
}

boolFilter.put("must", mustArray);

Map<String, Object> nestedBoolQuery = mapOf("bool", boolFilter);

Map<String, Object> topBoolFilter = mapOf("filter", nestedBoolQuery);

boolQuery.put("bool", topBoolFilter);

return boolQuery;
return boolFilter;
}
public static Map<String, Object> generateDSLQueryForLastCategory(String glossaryQualifiedName, String parentQualifiedName, boolean isTerm) {
public static Map<String, Object> generateDSLQueryForLastChild(String glossaryQualifiedName, String parentQualifiedName, boolean isTerm) {

Map<String, Object> sortKeyOrder = mapOf(LEXICOGRAPHICAL_SORT_ORDER, mapOf("order", "desc"));
Map<String, Object> scoreSortOrder = mapOf("_score", mapOf("order", "desc"));
Map<String, Object> displayNameSortOrder = mapOf("displayName.keyword", mapOf("order", "desc"));

Object[] sortArray = {sortKeyOrder, scoreSortOrder, displayNameSortOrder};
Object[] sortArray = {sortKeyOrder};

Map<String, Object> functionScore = mapOf("query", buildBoolQuery(glossaryQualifiedName, parentQualifiedName, isTerm));
Map<String, Object> boolMap = buildBoolQuery(glossaryQualifiedName, parentQualifiedName, isTerm);

Map<String, Object> dsl = new HashMap<>();
dsl.put("from", 0);
dsl.put("size", 1);
dsl.put("sort", sortArray);
dsl.put("query", mapOf("function_score", functionScore));
dsl.put("query", mapOf("bool", boolMap));

return dsl;
}

private static Map<String, Object> buildBoolQuery(String glossaryQualifiedName, String parentQualifiedName, boolean isTerm) {
Map<String, Object> boolQuery = new HashMap<>();
int mustArrayLength = 0;
if(StringUtils.isEmpty(parentQualifiedName) && StringUtils.isEmpty(glossaryQualifiedName)){
mustArrayLength = 2;
} else if(StringUtils.isEmpty(parentQualifiedName) && StringUtils.isNotEmpty(glossaryQualifiedName)){
mustArrayLength = 3;
} else {
mustArrayLength = 4;
}
Map<String, Object>[] mustArray = new Map[mustArrayLength];
Map<String, Object> boolFilter = new HashMap<>();
Map<String, Object>[] mustNotArray = new Map[2];
List<Map<String, Object>> mustArray = new ArrayList<>();
List<Map<String, Object>> mustNotArray = new ArrayList<>();

mustArray[0] = mapOf("term", mapOf("__state", "ACTIVE"));
mustArray.add(mapOf("term", mapOf("__state", "ACTIVE")));
if(StringUtils.isNotEmpty(glossaryQualifiedName)) {
String typeName = isTerm ? "AtlasGlossaryTerm" : "AtlasGlossaryCategory";
mustArray[1] = mapOf("term", mapOf("__typeName.keyword", typeName));
mustArray[2] = mapOf("term", mapOf("__glossary", glossaryQualifiedName));
mustArray.add(mapOf("term", mapOf("__typeName.keyword", typeName)));
mustArray.add(mapOf("term", mapOf("__glossary", glossaryQualifiedName)));
} else{
mustArray[1] = mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossary")));
mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossary"))));
}

if(StringUtils.isEmpty(parentQualifiedName)) {
mustNotArray[0] = mapOf("exists", mapOf("field", "__categories"));
mustNotArray[1] = mapOf("exists", mapOf("field", "__parentCategory"));
mustNotArray.add(mapOf("exists", mapOf("field", "__categories")));
mustNotArray.add(mapOf("exists", mapOf("field", "__parentCategory")));
boolFilter.put("must_not", mustNotArray);
}
else {
Map<String, Object>[] shouldParentArray = new Map[2];
shouldParentArray[0] = mapOf("term", mapOf("__categories", parentQualifiedName));
shouldParentArray[1] = mapOf("term", mapOf("__parentCategory", parentQualifiedName));
mustArray[3] = mapOf("bool",mapOf("should", shouldParentArray));
mustArray.add(mapOf("bool",mapOf("should", shouldParentArray)));
}

boolFilter.put("must", mustArray);

Map<String, Object> nestedBoolQuery = mapOf("bool", boolFilter);

Map<String, Object> topBoolFilter = mapOf("filter", nestedBoolQuery);

boolQuery.put("bool", topBoolFilter);

return boolQuery;
return boolFilter;
}
}
4 changes: 2 additions & 2 deletions ...rg/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,7 +134,7 @@ private void processCreateCategory(AtlasEntity entity, AtlasVertex vertex) throw
if(StringUtils.isEmpty(lexicographicalSortOrder)){
assignNewLexicographicalSortOrder(entity,glossaryQualifiedName, parentQname, this.discovery);
} else {
isValidLexoRank(lexicographicalSortOrder, glossaryQualifiedName, parentQname, this.discovery);
isValidLexoRank(false, lexicographicalSortOrder, glossaryQualifiedName, parentQname, this.discovery);
}

entity.setAttribute(QUALIFIED_NAME, createQualifiedName(vertex));
Expand Down Expand Up @@ -168,7 +168,7 @@ private void processUpdateCategory(AtlasEntity entity, AtlasVertex vertex) throw
parentQname = (String) parentCategory.getAttribute(QUALIFIED_NAME);
}
if(StringUtils.isNotEmpty(lexicographicalSortOrder)) {
isValidLexoRank(lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery);
isValidLexoRank(false, lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery);
}

if (!currentGlossaryQualifiedName.equals(newGlossaryQualifiedName)){
Expand Down
4 changes: 2 additions & 2 deletions ...rg/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,7 +95,7 @@ private void processCreateGlossary(AtlasStruct entity) throws AtlasBaseException
if(StringUtils.isEmpty(lexicographicalSortOrder)) {
assignNewLexicographicalSortOrder((AtlasEntity) entity, null, null, this.discovery);
} else {
isValidLexoRank(lexicographicalSortOrder, "", "", this.discovery);
isValidLexoRank(false, lexicographicalSortOrder, "", "", this.discovery);
}

entity.setAttribute(QUALIFIED_NAME, createQualifiedName());
Expand All @@ -116,7 +116,7 @@ private void processUpdateGlossary(AtlasStruct entity, AtlasVertex vertex) throw
}
String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER);
if(StringUtils.isNotEmpty(lexicographicalSortOrder)) {
isValidLexoRank(lexicographicalSortOrder, "", "", this.discovery);
isValidLexoRank(false, lexicographicalSortOrder, "", "", this.discovery);
}

String vertexQnName = vertex.getProperty(QUALIFIED_NAME, String.class);
Expand Down
4 changes: 2 additions & 2 deletions ...va/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ private void processCreateTerm(AtlasEntity entity, AtlasVertex vertex) throws At
if(StringUtils.isEmpty(lexicographicalSortOrder)){
assignNewLexicographicalSortOrder(entity, glossaryQName, parentQname, this.discovery);
} else {
isValidLexoRank(lexicographicalSortOrder, glossaryQName, parentQname, this.discovery);
isValidLexoRank(true, lexicographicalSortOrder, glossaryQName, parentQname, this.discovery);
}

entity.setAttribute(QUALIFIED_NAME, createQualifiedName());
Expand Down Expand Up @@ -137,7 +137,7 @@ private void processUpdateTerm(AtlasEntity entity, AtlasVertex vertex) throws At

String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER);
if(StringUtils.isNotEmpty(lexicographicalSortOrder)) {
isValidLexoRank(lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery);
isValidLexoRank(true, lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery);
}

if (!currentGlossaryQualifiedName.equals(newGlossaryQualifiedName)){
Expand Down
32 changes: 0 additions & 32 deletions repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem10.java
View file

This file was deleted.

0 comments on commit d5959d1

Please sign in to comment.