From cd985f6641b531c7b5094bbef1af33111a25c0e2 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Wed, 22 May 2024 10:55:08 +0530 Subject: [PATCH 01/13] Init commmit, added lexorank generation util lib and logic for appending the lexo attribute to category. --- .../java/org/apache/atlas/type/Constants.java | 2 + .../AbstractGlossaryPreProcessor.java | 115 ++++++- .../glossary/CategoryPreProcessor.java | 15 +- .../glossary/GlossaryPreProcessor.java | 7 +- .../atlas/util/lexoRank/LexoDecimal.java | 178 ++++++++++ .../atlas/util/lexoRank/LexoInteger.java | 325 ++++++++++++++++++ .../apache/atlas/util/lexoRank/LexoRank.java | 287 ++++++++++++++++ .../atlas/util/lexoRank/LexoRankBucket.java | 83 +++++ .../lexoRank/system/LexoNumeralSystem.java | 18 + .../lexoRank/system/LexoNumeralSystem10.java | 32 ++ .../lexoRank/system/LexoNumeralSystem36.java | 36 ++ .../lexoRank/system/LexoNumeralSystem64.java | 41 +++ 12 files changed, 1125 insertions(+), 14 deletions(-) create mode 100644 repository/src/main/java/org/apache/atlas/util/lexoRank/LexoDecimal.java create mode 100644 repository/src/main/java/org/apache/atlas/util/lexoRank/LexoInteger.java create mode 100644 repository/src/main/java/org/apache/atlas/util/lexoRank/LexoRank.java create mode 100644 repository/src/main/java/org/apache/atlas/util/lexoRank/LexoRankBucket.java create mode 100644 repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem.java create mode 100644 repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem10.java create mode 100644 repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem36.java create mode 100644 repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem64.java diff --git a/intg/src/main/java/org/apache/atlas/type/Constants.java b/intg/src/main/java/org/apache/atlas/type/Constants.java index 01550ae9c0..effc5208b1 100644 --- a/intg/src/main/java/org/apache/atlas/type/Constants.java +++ b/intg/src/main/java/org/apache/atlas/type/Constants.java @@ -54,10 +54,12 @@ public final class Constants { public static final String GLOSSARY_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "glossary"); public static final String CATEGORIES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "categories"); public static final String CATEGORIES_PARENT_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "parentCategory"); + public static final String MEANINGS_TEXT_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "meaningsText"); public static final String MEANING_NAMES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "meaningNames"); public static final String HAS_LINEAGE = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "hasLineage"); public static final String HAS_LINEAGE_VALID = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "hasLineageValid"); + public static final String LEXICOGRAPHICAL_SORT_ORDER = "lexicographicalSortOrder"; //Classification-Only System Attributes public static final String CLASSIFICATION_ENTITY_STATUS_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "entityStatus"); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java index 08c604489c..1d263ed8d8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java @@ -26,7 +26,6 @@ import org.apache.atlas.authorize.AtlasPrivilege; import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasObjectId; @@ -41,17 +40,14 @@ import org.apache.atlas.tasks.TaskManagement; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.util.lexoRank.LexoRank; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; import static org.apache.atlas.repository.Constants.ATLAS_GLOSSARY_TERM_ENTITY_TYPE; @@ -60,10 +56,8 @@ import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.indexSearchPaginated; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; -import static org.apache.atlas.type.Constants.MEANINGS_PROPERTY_KEY; -import static org.apache.atlas.type.Constants.MEANINGS_TEXT_PROPERTY_KEY; -import static org.apache.atlas.type.Constants.MEANING_NAMES_PROPERTY_KEY; -import static org.apache.atlas.type.Constants.PENDING_TASKS_PROPERTY_KEY; +import static org.apache.atlas.type.Constants.*; +import static org.apache.atlas.type.Constants.LEXICOGRAPHICAL_SORT_ORDER; public abstract class AbstractGlossaryPreProcessor implements PreProcessor { private static final Logger LOG = LoggerFactory.getLogger(AbstractGlossaryPreProcessor.class); @@ -73,6 +67,8 @@ public abstract class AbstractGlossaryPreProcessor implements PreProcessor { protected static final String ATTR_MEANINGS = "meanings"; protected static final String ATTR_CATEGORIES = "categories"; + protected static final String INIT_LEXORANK_OFFSET = "0|100000:"; + protected final AtlasTypeRegistry typeRegistry; protected final EntityGraphRetriever entityRetriever; protected final TaskManagement taskManagement; @@ -251,4 +247,103 @@ protected void recordUpdatedChildEntities(AtlasVertex entityVertex, Map attributes = new HashSet<>(); + attributes.add(LEXICOGRAPHICAL_SORT_ORDER); + List categories = null; + Map dslQuery = generateDSLQueryForLastCategory(glossaryQualifiedName, parentQualifiedName); + String lexoRank = ""; + try { + categories = indexSearchPaginated(dslQuery, attributes, this.discovery); + } catch (AtlasBaseException e) { + throw new RuntimeException(e); + } + + if (CollectionUtils.isNotEmpty(categories)) { + for (AtlasEntityHeader category : categories) { + String lexicographicalSortOrder = (String) category.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); + if(StringUtils.isNotEmpty(lexicographicalSortOrder)){ + LexoRank parsedLexoRank = LexoRank.parse(lexicographicalSortOrder); + LexoRank nextLexoRank = parsedLexoRank.genNext().genNext(); + lexoRank = nextLexoRank.toString(); + } else { + LexoRank parsedLexoRank = LexoRank.parse(INIT_LEXORANK_OFFSET); + LexoRank nextLexoRank = parsedLexoRank.genNext().genNext(); + lexoRank = nextLexoRank.toString(); + } + } + } else { + LexoRank parsedLexoRank = LexoRank.parse(INIT_LEXORANK_OFFSET); + LexoRank nextLexoRank = parsedLexoRank.genNext().genNext(); + lexoRank = nextLexoRank.toString(); + } + entity.setAttribute(LEXICOGRAPHICAL_SORT_ORDER, lexoRank); + } + + + public static Map generateDSLQueryForLastCategory(String glossaryQualifiedName, String parentQualifiedName) { + + Map sortKeyOrder = mapOf(LEXICOGRAPHICAL_SORT_ORDER, mapOf("order", "desc")); + Map scoreSortOrder = mapOf("_score", mapOf("order", "desc")); + Map displayNameSortOrder = mapOf("displayName.keyword", mapOf("order", "desc")); + + Object[] sortArray = {sortKeyOrder, scoreSortOrder, displayNameSortOrder}; + + Map functionScore = mapOf("query", buildBoolQuery(glossaryQualifiedName, parentQualifiedName)); + + Map dsl = new HashMap<>(); + dsl.put("from", 0); + dsl.put("size", 1); + dsl.put("sort", sortArray); + dsl.put("query", mapOf("function_score", functionScore)); + + return dsl; + } + + private static Map buildBoolQuery(String glossaryQualifiedName, String parentQualifiedName) { + Map boolQuery = new HashMap<>(); + int mustArrayLength = 0; + if(StringUtils.isEmpty(parentQualifiedName) && StringUtils.isEmpty(glossaryQualifiedName)){ + mustArrayLength = 2; + } else if(StringUtils.isEmpty(parentQualifiedName) && StringUtils.isNotEmpty(glossaryQualifiedName)){ + mustArrayLength = 3; + } else { + mustArrayLength = 4; + } + Map[] mustArray = new Map[mustArrayLength]; + Map boolFilter = new HashMap<>(); + Map[] mustNotArray = new Map[2]; + + mustArray[0] = mapOf("term", mapOf("__state", "ACTIVE")); + if(StringUtils.isNotEmpty(glossaryQualifiedName)) { + mustArray[1] = mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossaryTerm", "AtlasGlossaryCategory"))); + mustArray[2] = mapOf("term", mapOf("__glossary", glossaryQualifiedName)); + } else{ + mustArray[1] = mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossary"))); + } + + if(StringUtils.isEmpty(parentQualifiedName)) { + mustNotArray[0] = mapOf("exists", mapOf("field", "__categories")); + mustNotArray[1] = mapOf("exists", mapOf("field", "__parentCategory")); + boolFilter.put("must_not", mustNotArray); + } + else { + Map[] shouldParentArray = new Map[2]; + shouldParentArray[0] = mapOf("term", mapOf("__categories", parentQualifiedName)); + shouldParentArray[1] = mapOf("term", mapOf("__parentCategory", parentQualifiedName)); + mustArray[3] = mapOf("bool",mapOf("should", shouldParentArray)); + } + + boolFilter.put("must", mustArray); + + Map nestedBoolQuery = mapOf("bool", boolFilter); + + Map topBoolFilter = mapOf("filter", nestedBoolQuery); + + boolQuery.put("bool", topBoolFilter); + + return boolQuery; + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java index 88f72d2f16..532538aa1b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java @@ -40,6 +40,7 @@ import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasStructType; import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.util.lexoRank.LexoRank; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; @@ -47,6 +48,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.*; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -70,9 +72,7 @@ import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.*; import static org.apache.atlas.repository.store.graph.v2.tasks.MeaningsTaskFactory.UPDATE_ENTITY_MEANINGS_ON_TERM_UPDATE; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; -import static org.apache.atlas.type.Constants.CATEGORIES_PARENT_PROPERTY_KEY; -import static org.apache.atlas.type.Constants.CATEGORIES_PROPERTY_KEY; -import static org.apache.atlas.type.Constants.GLOSSARY_PROPERTY_KEY; +import static org.apache.atlas.type.Constants.*; public class CategoryPreProcessor extends AbstractGlossaryPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(CategoryPreProcessor.class); @@ -117,12 +117,20 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co private void processCreateCategory(AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreateCategory"); String catName = (String) entity.getAttribute(NAME); + String parentQname = null; if (StringUtils.isEmpty(catName) || isNameInvalid(catName)) { throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); } String glossaryQualifiedName = (String) anchor.getAttribute(QUALIFIED_NAME); + if (parentCategory != null) { + parentQname = (String) parentCategory.getAttribute(QUALIFIED_NAME); + } + String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); + if(StringUtils.isEmpty(lexicographicalSortOrder)){ + assignNewLexicographicalSortOrder(entity,glossaryQualifiedName, parentQname); + } categoryExists(catName, glossaryQualifiedName); validateParent(glossaryQualifiedName); @@ -489,4 +497,5 @@ private String createQualifiedName(AtlasVertex vertex) { return getUUID() + "@" + anchor.getAttribute(QUALIFIED_NAME); } + } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java index 6e3c962426..5cea0d0683 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java @@ -39,6 +39,7 @@ import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.getUUID; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.isNameInvalid; +import static org.apache.atlas.type.Constants.LEXICOGRAPHICAL_SORT_ORDER; public class GlossaryPreProcessor implements PreProcessor { private static final Logger LOG = LoggerFactory.getLogger(GlossaryPreProcessor.class); @@ -77,7 +78,11 @@ private void processCreateGlossary(AtlasStruct entity) throws AtlasBaseException if (StringUtils.isEmpty(glossaryName) || isNameInvalid(glossaryName)) { throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); } - + String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); + // TODO : Figure the placement for this method for glossary creation. +// if(StringUtils.isEmpty(lexicographicalSortOrder)){ +// assignNewLexicographicalSortOrder(entity,null, null); +// } if (glossaryExists(glossaryName)) { throw new AtlasBaseException(AtlasErrorCode.GLOSSARY_ALREADY_EXISTS,glossaryName); } diff --git a/repository/src/main/java/org/apache/atlas/util/lexoRank/LexoDecimal.java b/repository/src/main/java/org/apache/atlas/util/lexoRank/LexoDecimal.java new file mode 100644 index 0000000000..195970660a --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/util/lexoRank/LexoDecimal.java @@ -0,0 +1,178 @@ +package org.apache.atlas.util.lexoRank; + +import org.apache.atlas.util.lexoRank.system.LexoNumeralSystem; + +import java.util.Objects; + +public class LexoDecimal implements Comparable { + + private final LexoInteger mag; + private final int sig; + + private LexoDecimal(LexoInteger mag, int sig) { + this.mag = mag; + this.sig = sig; + } + + public static LexoDecimal half(LexoNumeralSystem sys) { + int mid = sys.getBase() / 2; + return make(LexoInteger.make(sys, 1, new int[] {mid}), 1); + } + + public static LexoDecimal parse(String str, LexoNumeralSystem system) { + int partialIndex = str.indexOf(system.getRadixPointChar()); + if (str.lastIndexOf(system.getRadixPointChar()) != partialIndex) + throw new IllegalArgumentException("More than one " + system.getRadixPointChar()); + + if (partialIndex < 0) return make(LexoInteger.parse(str, system), 0); + + String intStr = str.substring(0, partialIndex) + str.substring(partialIndex + 1); + return make(LexoInteger.parse(intStr, system), str.length() - 1 - partialIndex); + } + + public static LexoDecimal from(LexoInteger integer) { + return make(integer, 0); + } + + public static LexoDecimal make(LexoInteger integer, int sig) { + if (integer.isZero()) return new LexoDecimal(integer, 0); + + int zeroCount = 0; + + for (int i = 0; i < sig && integer.getMag(i) == 0; ++i) ++zeroCount; + + LexoInteger newInteger = integer.shiftRight(zeroCount); + int newSig = sig - zeroCount; + return new LexoDecimal(newInteger, newSig); + } + + public LexoNumeralSystem getSystem() { + return mag.getSystem(); + } + + public LexoDecimal add(LexoDecimal other) { + LexoInteger tMag = mag; + int tSig = sig; + LexoInteger oMag = other.mag; + + int oSig; + for (oSig = other.sig; tSig < oSig; ++tSig) tMag = tMag.shiftLeft(); + + while (tSig > oSig) { + oMag = oMag.shiftLeft(); + ++oSig; + } + + return make(tMag.add(oMag), tSig); + } + + public LexoDecimal subtract(LexoDecimal other) { + LexoInteger thisMag = mag; + int thisSig = sig; + LexoInteger otherMag = other.mag; + + int otherSig; + for (otherSig = other.sig; thisSig < otherSig; ++thisSig) thisMag = thisMag.shiftLeft(); + + while (thisSig > otherSig) { + otherMag = otherMag.shiftLeft(); + ++otherSig; + } + + return make(thisMag.subtract(otherMag), thisSig); + } + + public LexoDecimal multiply(LexoDecimal other) { + return make(mag.multiply(other.mag), sig + other.sig); + } + + public LexoInteger floor() { + return mag.shiftRight(sig); + } + + public LexoInteger ceil() { + if (isExact()) return mag; + + LexoInteger floor = floor(); + return floor.add(LexoInteger.one(floor.getSystem())); + } + + public boolean isExact() { + if (sig == 0) return true; + + for (int i = 0; i < sig; ++i) if (mag.getMag(i) != 0) return false; + + return true; + } + + public int getScale() { + return sig; + } + + public LexoDecimal setScale(int nSig) { + return setScale(nSig, false); + } + + public LexoDecimal setScale(int nSig, boolean ceiling) { + if (nSig >= sig) return this; + + if (nSig < 0) nSig = 0; + + int diff = sig - nSig; + LexoInteger nmag = mag.shiftRight(diff); + if (ceiling) nmag = nmag.add(LexoInteger.one(nmag.getSystem())); + + return make(nmag, nSig); + } + + public String format() { + String intStr = mag.format(); + if (sig == 0) return intStr; + + StringBuilder sb = new StringBuilder(intStr); + char head = sb.charAt(0); + boolean specialHead = + head == mag.getSystem().getPositiveChar() || head == mag.getSystem().getNegativeChar(); + if (specialHead) sb.delete(0, 1); + + while (sb.length() < sig + 1) sb.insert(0, mag.getSystem().toChar(0)); + + sb.insert(sb.length() - sig, mag.getSystem().getRadixPointChar()); + if (sb.length() - sig == 0) sb.insert(0, mag.getSystem().toChar(0)); + + if (specialHead) sb.insert(0, head); + + return sb.toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + LexoDecimal that = (LexoDecimal) o; + return sig == that.sig && Objects.equals(mag, that.mag); + } + + @Override + public int hashCode() { + return Objects.hash(mag, sig); + } + + @Override + public String toString() { + return format(); + } + + @Override + public int compareTo(LexoDecimal lexoDecimal) { + if (Objects.equals(this, lexoDecimal)) return 0; + if (Objects.equals(null, lexoDecimal)) return 1; + + LexoInteger tMag = mag; + LexoInteger oMag = lexoDecimal.mag; + if (sig > lexoDecimal.sig) oMag = oMag.shiftLeft(sig - lexoDecimal.sig); + else if (sig < lexoDecimal.sig) tMag = tMag.shiftLeft(lexoDecimal.sig - sig); + + return tMag.compareTo(oMag); + } +} diff --git a/repository/src/main/java/org/apache/atlas/util/lexoRank/LexoInteger.java b/repository/src/main/java/org/apache/atlas/util/lexoRank/LexoInteger.java new file mode 100644 index 0000000000..187797a20b --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/util/lexoRank/LexoInteger.java @@ -0,0 +1,325 @@ +package org.apache.atlas.util.lexoRank; + + +import org.apache.atlas.util.lexoRank.system.LexoNumeralSystem; + +import java.util.Arrays; +import java.util.Objects; + +public class LexoInteger implements Comparable { + private static final int[] ZERO_MAG = {0}; + private static final int[] ONE_MAG = {1}; + private final int negativeSign = -1; + private final int zeroSign = 0; + private final int positiveSign = 1; + private final int[] mag; + private final int sign; + private final LexoNumeralSystem sys; + + private LexoInteger(LexoNumeralSystem system, int sign, int[] mag) { + sys = system; + this.sign = sign; + this.mag = mag; + } + + private static int[] add(LexoNumeralSystem sys, int[] l, int[] r) { + int estimatedSize = Math.max(l.length, r.length); + int[] result = new int[estimatedSize]; + int carry = 0; + + for (int i = 0; i < estimatedSize; ++i) { + int lNum = i < l.length ? l[i] : 0; + int rNum = i < r.length ? r[i] : 0; + int sum = lNum + rNum + carry; + + for (carry = 0; sum >= sys.getBase(); sum -= sys.getBase()) ++carry; + + result[i] = sum; + } + + return extendWithCarry(result, carry); + } + + private static int[] extendWithCarry(int[] mag, int carry) { + int[] result = mag; + if (carry > 0) { + int[] extendedMag = new int[mag.length + 1]; + System.arraycopy(mag, 0, extendedMag, 0, mag.length); + extendedMag[extendedMag.length - 1] = carry; + result = extendedMag; + } + + return result; + } + + private static int[] subtract(LexoNumeralSystem sys, int[] l, int[] r) { + int[] rComplement = complement(sys, r, l.length); + int[] rSum = add(sys, l, rComplement); + rSum[rSum.length - 1] = 0; + return add(sys, rSum, ONE_MAG); + } + + private static int[] multiply(LexoNumeralSystem sys, int[] l, int[] r) { + int[] result = new int[l.length + r.length]; + + for (int li = 0; li < l.length; ++li) + for (int ri = 0; ri < r.length; ++ri) { + int resultIndex = li + ri; + + for (result[resultIndex] += l[li] * r[ri]; + result[resultIndex] >= sys.getBase(); + result[resultIndex] -= sys.getBase()) ++result[resultIndex + 1]; + } + + return result; + } + + private static int[] complement(LexoNumeralSystem sys, int[] mag, int digits) { + if (digits <= 0) throw new IllegalArgumentException("Expected at least 1 digit"); + + int[] nmag = new int[digits]; + + Arrays.fill(nmag, sys.getBase() - 1); + + for (int i = 0; i < mag.length; ++i) nmag[i] = sys.getBase() - 1 - mag[i]; + + return nmag; + } + + private static int compare(int[] l, int[] r) { + if (l.length < r.length) return -1; + + if (l.length > r.length) return 1; + + for (int i = l.length - 1; i >= 0; --i) { + if (l[i] < r[i]) return -1; + + if (l[i] > r[i]) return 1; + } + + return 0; + } + + public static LexoInteger parse(String strFull, LexoNumeralSystem system) { + String str = strFull; + int sign = 1; + if (strFull.indexOf(system.getPositiveChar()) == 0) { + str = strFull.substring(1); + } else if (strFull.indexOf(system.getNegativeChar()) == 0) { + str = strFull.substring(1); + sign = -1; + } + + int[] mag = new int[str.length()]; + int strIndex = mag.length - 1; + + for (int magIndex = 0; strIndex >= 0; ++magIndex) { + mag[magIndex] = system.toDigit(str.charAt(strIndex)); + --strIndex; + } + + return make(system, sign, mag); + } + + protected static LexoInteger zero(LexoNumeralSystem sys) { + return new LexoInteger(sys, 0, ZERO_MAG); + } + + protected static LexoInteger one(LexoNumeralSystem sys) { + return make(sys, 1, ONE_MAG); + } + + public static LexoInteger make(LexoNumeralSystem sys, int sign, int[] mag) { + int actualLength; + actualLength = mag.length; + while (actualLength > 0 && mag[actualLength - 1] == 0) { + --actualLength; + } + + if (actualLength == 0) return zero(sys); + + if (actualLength == mag.length) return new LexoInteger(sys, sign, mag); + + int[] nmag = new int[actualLength]; + System.arraycopy(mag, 0, nmag, 0, actualLength); + return new LexoInteger(sys, sign, nmag); + } + + public LexoInteger add(LexoInteger other) { + checkSystem(other); + if (isZero()) return other; + + if (other.isZero()) return this; + + if (sign != other.sign) { + LexoInteger pos; + if (sign == -1) { + pos = negate(); + LexoInteger val = pos.subtract(other); + return val.negate(); + } + + pos = other.negate(); + return subtract(pos); + } + + int[] result = add(sys, mag, other.mag); + return make(sys, sign, result); + } + + public LexoInteger subtract(LexoInteger other) { + checkSystem(other); + if (isZero()) return other.negate(); + + if (other.isZero()) return this; + + if (sign != other.sign) { + LexoInteger negate; + if (sign == -1) { + negate = negate(); + LexoInteger sum = negate.add(other); + return sum.negate(); + } + + negate = other.negate(); + return add(negate); + } + + int cmp = compare(mag, other.mag); + if (cmp == 0) return zero(sys); + + return cmp < 0 + ? make(sys, sign == -1 ? 1 : -1, subtract(sys, other.mag, mag)) + : make(sys, sign == -1 ? -1 : 1, subtract(sys, mag, other.mag)); + } + + public LexoInteger multiply(LexoInteger other) { + checkSystem(other); + if (isZero()) return this; + + if (other.isZero()) return other; + + if (isOneish()) return sign == other.sign ? make(sys, 1, other.mag) : make(sys, -1, other.mag); + + if (other.isOneish()) return sign == other.sign ? make(sys, 1, mag) : make(sys, -1, mag); + + int[] newMag = multiply(sys, mag, other.mag); + return sign == other.sign ? make(sys, 1, newMag) : make(sys, -1, newMag); + } + + public LexoInteger negate() { + return isZero() ? this : make(sys, sign == 1 ? -1 : 1, mag); + } + + public LexoInteger shiftLeft() { + return shiftLeft(1); + } + + public LexoInteger shiftLeft(int times) { + if (times == 0) return this; + + if (times < 0) return shiftRight(Math.abs(times)); + + int[] nmag = new int[mag.length + times]; + System.arraycopy(mag, 0, nmag, times, mag.length); + return make(sys, sign, nmag); + } + + public LexoInteger shiftRight() { + return shiftRight(1); + } + + public LexoInteger shiftRight(int times) { + if (mag.length - times <= 0) return zero(sys); + + int[] nmag = new int[mag.length - times]; + System.arraycopy(mag, times, nmag, 0, nmag.length); + return make(sys, sign, nmag); + } + + public LexoInteger complement() { + return complement(mag.length); + } + + private LexoInteger complement(int digits) { + return make(sys, sign, complement(sys, mag, digits)); + } + + public boolean isZero() { + return sign == 0 && mag.length == 1 && mag[0] == 0; + } + + private boolean isOneish() { + return mag.length == 1 && mag[0] == 1; + } + + public boolean isOne() { + return sign == 1 && mag.length == 1 && mag[0] == 1; + } + + public int getMag(int index) { + return mag[index]; + } + + public LexoNumeralSystem getSystem() { + return sys; + } + + private void checkSystem(LexoInteger other) { + if (!sys.getName().equals(other.sys.getName())) + throw new IllegalArgumentException("Expected numbers of same numeral sys"); + } + + public String format() { + if (isZero()) return String.valueOf(sys.toChar(0)); + StringBuilder sb = new StringBuilder(); + for (int digit : mag) { + sb.insert(0, sys.toChar(digit)); + } + if (sign == -1) sb.setCharAt(0, sys.getNegativeChar()); + + return sb.toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + LexoInteger that = (LexoInteger) o; + return sign == that.sign && Arrays.equals(mag, that.mag) && Objects.equals(sys, that.sys); + } + + @Override + public int hashCode() { + int result = Objects.hash(negativeSign, zeroSign, positiveSign, sign, sys); + result = 31 * result + Arrays.hashCode(mag); + return result; + } + + @Override + public String toString() { + return format(); + } + + @Override + public int compareTo(LexoInteger lexoInteger) { + if (this.equals(lexoInteger)) return 0; + if (null == lexoInteger) return 1; + + if (sign == -1) { + if (lexoInteger.sign == -1) { + int cmp = compare(mag, lexoInteger.mag); + if (cmp == -1) return 1; + return cmp == 1 ? -1 : 0; + } + + return -1; + } + + if (sign == 1) return lexoInteger.sign == 1 ? compare(mag, lexoInteger.mag) : 1; + + if (lexoInteger.sign == -1) return 1; + + return lexoInteger.sign == 1 ? -1 : 0; + } +} diff --git a/repository/src/main/java/org/apache/atlas/util/lexoRank/LexoRank.java b/repository/src/main/java/org/apache/atlas/util/lexoRank/LexoRank.java new file mode 100644 index 0000000000..29a8bf2d41 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/util/lexoRank/LexoRank.java @@ -0,0 +1,287 @@ +package org.apache.atlas.util.lexoRank; + + + + +import org.apache.atlas.util.lexoRank.system.LexoNumeralSystem; +import org.apache.atlas.util.lexoRank.system.LexoNumeralSystem36; + +import java.util.Objects; + +public class LexoRank implements Comparable { + + public static final LexoNumeralSystem NUMERAL_SYSTEM = new LexoNumeralSystem36(); + private static final LexoDecimal ZERO_DECIMAL = LexoDecimal.parse("0", NUMERAL_SYSTEM); + private static final LexoDecimal ONE_DECIMAL = LexoDecimal.parse("1", NUMERAL_SYSTEM); + private static final LexoDecimal EIGHT_DECIMAL = LexoDecimal.parse("8", NUMERAL_SYSTEM); + private static final LexoDecimal MIN_DECIMAL = ZERO_DECIMAL; + + private static final LexoDecimal MAX_DECIMAL = + LexoDecimal.parse("1000000", NUMERAL_SYSTEM).subtract(ONE_DECIMAL); + + private static final LexoDecimal MID_DECIMAL = between(MIN_DECIMAL, MAX_DECIMAL); + private static final LexoDecimal INITIAL_MIN_DECIMAL = + LexoDecimal.parse("100000", NUMERAL_SYSTEM); + + private static final LexoDecimal INITIAL_MAX_DECIMAL = + LexoDecimal.parse( + NUMERAL_SYSTEM.toChar(NUMERAL_SYSTEM.getBase() - 2) + "00000", NUMERAL_SYSTEM); + + private final String value; + private final LexoRankBucket bucket; + private final LexoDecimal decimal; + + private LexoRank(String value) { + this.value = value; + String[] parts = this.value.split("\\|"); + bucket = LexoRankBucket.from(parts[0]); + decimal = LexoDecimal.parse(parts[1], NUMERAL_SYSTEM); + } + + private LexoRank(LexoRankBucket bucket, LexoDecimal dec) { + value = bucket.format() + "|" + formatDecimal(dec); + this.bucket = bucket; + decimal = dec; + } + + public static LexoRank min() { + return from(LexoRankBucket.BUCKET_0, MIN_DECIMAL); + } + + public static LexoRank max() { + return max(LexoRankBucket.BUCKET_0); + } + + public static LexoRank middle() { + LexoRank minLexoRank = min(); + return minLexoRank.between(max(minLexoRank.bucket)); + } + + public static LexoRank max(LexoRankBucket bucket) { + return from(bucket, MAX_DECIMAL); + } + + public static LexoRank initial(LexoRankBucket bucket) { + return bucket == LexoRankBucket.BUCKET_0 + ? from(bucket, INITIAL_MIN_DECIMAL) + : from(bucket, INITIAL_MAX_DECIMAL); + } + + private static LexoDecimal between(LexoDecimal oLeft, LexoDecimal oRight) { + if (oLeft.getSystem() != oRight.getSystem()) + throw new IllegalArgumentException("Expected same system"); + + LexoDecimal left = oLeft; + LexoDecimal right = oRight; + LexoDecimal nLeft; + if (oLeft.getScale() < oRight.getScale()) { + nLeft = oRight.setScale(oLeft.getScale(), false); + if (oLeft.compareTo(nLeft) >= 0) return middle(oLeft, oRight); + + right = nLeft; + } + + if (oLeft.getScale() > right.getScale()) { + nLeft = oLeft.setScale(right.getScale(), true); + if (nLeft.compareTo(right) >= 0) return middle(oLeft, oRight); + + left = nLeft; + } + + LexoDecimal nRight; + for (int scale = left.getScale(); scale > 0; right = nRight) { + int nScale1 = scale - 1; + LexoDecimal nLeft1 = left.setScale(nScale1, true); + nRight = right.setScale(nScale1, false); + int cmp = nLeft1.compareTo(nRight); + if (cmp == 0) return checkMid(oLeft, oRight, nLeft1); + + if (nLeft1.compareTo(nRight) > 0) break; + + scale = nScale1; + left = nLeft1; + } + + LexoDecimal mid = middle(oLeft, oRight, left, right); + + int nScale; + for (int mScale = mid.getScale(); mScale > 0; mScale = nScale) { + nScale = mScale - 1; + LexoDecimal nMid = mid.setScale(nScale); + if (oLeft.compareTo(nMid) >= 0 || nMid.compareTo(oRight) >= 0) break; + + mid = nMid; + } + + return mid; + } + + private static LexoDecimal middle( + LexoDecimal lBound, LexoDecimal rBound, LexoDecimal left, LexoDecimal right) { + LexoDecimal mid = middle(left, right); + return checkMid(lBound, rBound, mid); + } + + private static LexoDecimal checkMid(LexoDecimal lBound, LexoDecimal rBound, LexoDecimal mid) { + if (lBound.compareTo(mid) >= 0) return middle(lBound, rBound); + + return mid.compareTo(rBound) >= 0 ? middle(lBound, rBound) : mid; + } + + private static LexoDecimal middle(LexoDecimal left, LexoDecimal right) { + LexoDecimal sum = left.add(right); + LexoDecimal mid = sum.multiply(LexoDecimal.half(left.getSystem())); + int scale = Math.max(left.getScale(), right.getScale()); + if (mid.getScale() > scale) { + LexoDecimal roundDown = mid.setScale(scale, false); + if (roundDown.compareTo(left) > 0) return roundDown; + + LexoDecimal roundUp = mid.setScale(scale, true); + if (roundUp.compareTo(right) < 0) return roundUp; + } + + return mid; + } + + private static String formatDecimal(LexoDecimal dec) { + String formatVal = dec.format(); + StringBuilder val = new StringBuilder(formatVal); + int partialIndex = formatVal.indexOf(NUMERAL_SYSTEM.getRadixPointChar()); + char zero = NUMERAL_SYSTEM.toChar(0); + if (partialIndex < 0) { + partialIndex = formatVal.length(); + val.append(NUMERAL_SYSTEM.getRadixPointChar()); + } + + while (partialIndex < 6) { + val.insert(0, zero); + ++partialIndex; + } + + // TODO CHECK LOGIC + int valLength = val.length() - 1; + while (val.charAt(valLength) == zero) { + valLength = val.length() - 1; + } + + return val.toString(); + } + + public static LexoRank parse(String str) { + if (isNullOrWhiteSpace(str)) throw new IllegalArgumentException(str); + return new LexoRank(str); + } + + public static LexoRank from(LexoRankBucket bucket, LexoDecimal dec) { + if (!dec.getSystem().getName().equals(NUMERAL_SYSTEM.getName())) + throw new IllegalArgumentException("Expected different system"); + + return new LexoRank(bucket, dec); + } + + private static boolean isNullOrWhiteSpace(String string) { + return string == null || string.equals(" "); + } + + public LexoRankBucket getBucket() { + return bucket; + } + + public LexoDecimal getDecimal() { + return decimal; + } + + public int CompareTo(LexoRank other) { + if (Objects.equals(this, other)) return 0; + if (Objects.equals(null, other)) return 1; + return value.compareTo(other.value); + } + + public LexoRank genPrev() { + if (isMax()) return new LexoRank(bucket, INITIAL_MAX_DECIMAL); + + LexoInteger floorInteger = decimal.floor(); + LexoDecimal floorDecimal = LexoDecimal.from(floorInteger); + LexoDecimal nextDecimal = floorDecimal.subtract(EIGHT_DECIMAL); + if (nextDecimal.compareTo(MIN_DECIMAL) <= 0) nextDecimal = between(MIN_DECIMAL, decimal); + + return new LexoRank(bucket, nextDecimal); + } + + public LexoRank inNextBucket() { + return from(bucket.next(), decimal); + } + + public LexoRank inPrevBucket() { + return from(bucket.prev(), decimal); + } + + public boolean isMin() { + return decimal.equals(MIN_DECIMAL); + } + + public boolean isMax() { + return decimal.equals(MAX_DECIMAL); + } + + public String format() { + return value; + } + + public LexoRank genNext() { + if (isMin()) return new LexoRank(bucket, INITIAL_MIN_DECIMAL); + + LexoInteger ceilInteger = decimal.ceil(); + LexoDecimal ceilDecimal = LexoDecimal.from(ceilInteger); + LexoDecimal nextDecimal = ceilDecimal.add(EIGHT_DECIMAL); + if (nextDecimal.compareTo(MAX_DECIMAL) >= 0) nextDecimal = between(decimal, MAX_DECIMAL); + + return new LexoRank(bucket, nextDecimal); + } + + public LexoRank between(LexoRank other) { + if (!bucket.equals(other.bucket)) + throw new IllegalArgumentException("Between works only within the same bucket"); + + int cmp = decimal.compareTo(other.decimal); + if (cmp > 0) return new LexoRank(bucket, between(other.decimal, decimal)); + if (cmp == 0) + throw new IllegalArgumentException( + "Try to rank between issues with same rank this=" + + this + + " other=" + + other + + " this.decimal=" + + decimal + + " other.decimal=" + + other.decimal); + return new LexoRank(bucket, between(decimal, other.decimal)); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + LexoRank lexoRank = (LexoRank) o; + return Objects.equals(value, lexoRank.value) + && Objects.equals(bucket, lexoRank.bucket) + && Objects.equals(decimal, lexoRank.decimal); + } + + @Override + public int hashCode() { + return Objects.hash(value, bucket, decimal); + } + + @Override + public String toString() { + return format(); + } + + @Override + public int compareTo(LexoRank lexoRank) { + if (Objects.equals(this, lexoRank)) return 0; + if (Objects.equals(null, lexoRank)) return 1; + return value.compareTo(lexoRank.value); + } +} diff --git a/repository/src/main/java/org/apache/atlas/util/lexoRank/LexoRankBucket.java b/repository/src/main/java/org/apache/atlas/util/lexoRank/LexoRankBucket.java new file mode 100644 index 0000000000..c3b60b4982 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/util/lexoRank/LexoRankBucket.java @@ -0,0 +1,83 @@ +package org.apache.atlas.util.lexoRank; + + +import java.util.Objects; + +public class LexoRankBucket { + + protected static final LexoRankBucket BUCKET_0 = new LexoRankBucket("0"); + protected static final LexoRankBucket BUCKET_1 = new LexoRankBucket("1"); + protected static final LexoRankBucket BUCKET_2 = new LexoRankBucket("2"); + + private static final LexoRankBucket[] VALUES = {BUCKET_0, BUCKET_1, BUCKET_2}; + + private final LexoInteger value; + + private LexoRankBucket(String val) { + value = LexoInteger.parse(val, LexoRank.NUMERAL_SYSTEM); + } + + public static LexoRankBucket resolve(int bucketId) { + for (LexoRankBucket bucket : VALUES) { + if (bucket.equals(from(String.valueOf(bucketId)))) return bucket; + } + + throw new IllegalArgumentException("No bucket found with id " + bucketId); + } + + public static LexoRankBucket from(String str) { + LexoInteger val = LexoInteger.parse(str, LexoRank.NUMERAL_SYSTEM); + + for (LexoRankBucket bucket : VALUES) { + if (bucket.value.equals(val)) return bucket; + } + + throw new IllegalArgumentException("Unknown bucket: " + str); + } + + public static LexoRankBucket min() { + return VALUES[0]; + } + + public static LexoRankBucket max() { + return VALUES[VALUES.length - 1]; + } + + public String format() { + return value.format(); + } + + public LexoRankBucket next() { + if (this == BUCKET_0) return BUCKET_1; + + if (this == BUCKET_1) return BUCKET_2; + + return this == BUCKET_2 ? BUCKET_0 : BUCKET_2; + } + + public LexoRankBucket prev() { + if (this == BUCKET_0) return BUCKET_2; + + if (this == BUCKET_1) return BUCKET_0; + + return this == BUCKET_2 ? BUCKET_1 : BUCKET_0; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + LexoRankBucket that = (LexoRankBucket) o; + return Objects.equals(value, that.value); + } + + @Override + public String toString() { + return format(); + } + + @Override + public int hashCode() { + return Objects.hash(value); + } +} diff --git a/repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem.java b/repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem.java new file mode 100644 index 0000000000..a996079e3d --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem.java @@ -0,0 +1,18 @@ +package org.apache.atlas.util.lexoRank.system; + +public interface LexoNumeralSystem { + + String getName(); + + int getBase(); + + char getPositiveChar(); + + char getNegativeChar(); + + char getRadixPointChar(); + + int toDigit(char var1); + + char toChar(int var1); +} diff --git a/repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem10.java b/repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem10.java new file mode 100644 index 0000000000..f2c84e1a50 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem10.java @@ -0,0 +1,32 @@ +package org.apache.atlas.util.lexoRank.system; + +public class LexoNumeralSystem10 implements LexoNumeralSystem { + public String getName() { + return "Base10"; + } + + public int getBase() { + return 10; + } + + public char getPositiveChar() { + return '+'; + } + + public char getNegativeChar() { + return '-'; + } + + public char getRadixPointChar() { + return '.'; + } + + public int toDigit(char ch) { + if (ch >= '0' && ch <= '9') return ch - 48; + throw new IllegalArgumentException("Not valid digit: " + ch); + } + + public char toChar(int digit) { + return (char) (digit + 48); + } +} diff --git a/repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem36.java b/repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem36.java new file mode 100644 index 0000000000..731871842e --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem36.java @@ -0,0 +1,36 @@ +package org.apache.atlas.util.lexoRank.system; + + +public class LexoNumeralSystem36 implements LexoNumeralSystem { + private final char[] digits = "0123456789abcdefghijklmnopqrstuvwxyz".toCharArray(); + + public String getName() { + return "Base36"; + } + + public int getBase() { + return 36; + } + + public char getPositiveChar() { + return '+'; + } + + public char getNegativeChar() { + return '-'; + } + + public char getRadixPointChar() { + return ':'; + } + + public int toDigit(char ch) { + if (ch >= '0' && ch <= '9') return ch - 48; + if (ch >= 'a' && ch <= 'z') return ch - 97 + 10; + throw new IllegalArgumentException("Not valid digit: " + ch); + } + + public char toChar(int digit) { + return digits[digit]; + } +} diff --git a/repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem64.java b/repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem64.java new file mode 100644 index 0000000000..89b0af2390 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem64.java @@ -0,0 +1,41 @@ +package org.apache.atlas.util.lexoRank.system; + + +public class LexoNumeralSystem64 implements LexoNumeralSystem { + + private final char[] digits = + "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ^_abcdefghijklmnopqrstuvwxyz".toCharArray(); + + public String getName() { + return "Base64"; + } + + public int getBase() { + return 64; + } + + public char getPositiveChar() { + return '+'; + } + + public char getNegativeChar() { + return '-'; + } + + public char getRadixPointChar() { + return ':'; + } + + public int toDigit(char ch) { + if (ch >= '0' && ch <= '9') return ch - 48; + if (ch >= 'A' && ch <= 'Z') return ch - 65 + 10; + if (ch == '^') return 36; + if (ch == '_') return 37; + if (ch >= 'a' && ch <= 'z') return ch - 97 + 38; + throw new IllegalArgumentException("Not valid digit: " + ch); + } + + public char toChar(int digit) { + return digits[digit]; + } +} From 0bdee872c6d9065dfddb96dbfd5f7ded8ab8c7ac Mon Sep 17 00:00:00 2001 From: hr2904 Date: Wed, 22 May 2024 18:19:36 +0530 Subject: [PATCH 02/13] Added method to append lexicographicSortOrder attribute to any Glossary based Entity that does not have it while creation. --- .../store/graph/v2/AtlasEntityStoreV2.java | 2 +- .../v2/preprocessor/PreProcessorUtils.java | 118 ++++++++++++++++++ .../AbstractGlossaryPreProcessor.java | 100 --------------- .../glossary/CategoryPreProcessor.java | 7 +- .../glossary/GlossaryPreProcessor.java | 23 ++-- .../glossary/TermPreProcessor.java | 17 ++- .../java/org/apache/atlas/RequestContext.java | 12 ++ 7 files changed, 164 insertions(+), 115 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 83df0fe2ef..f98f91b821 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -1821,7 +1821,7 @@ public PreProcessor getPreProcessor(String typeName) { switch (typeName) { case ATLAS_GLOSSARY_ENTITY_TYPE: - preProcessor = new GlossaryPreProcessor(typeRegistry, entityRetriever); + preProcessor = new GlossaryPreProcessor(typeRegistry, entityRetriever, graph); break; case ATLAS_GLOSSARY_TERM_ENTITY_TYPE: diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index 2680e48207..53380100cb 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -1,6 +1,7 @@ package org.apache.atlas.repository.store.graph.v2.preprocessor; import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.RequestContext; import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.discovery.IndexSearchParams; @@ -14,6 +15,7 @@ import org.apache.atlas.type.AtlasStructType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.util.NanoIdUtils; +import org.apache.atlas.util.lexoRank.LexoRank; import org.apache.atlas.utils.AtlasEntityUtil; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; @@ -26,6 +28,7 @@ import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; +import static org.apache.atlas.type.Constants.LEXICOGRAPHICAL_SORT_ORDER; public class PreProcessorUtils { private static final Logger LOG = LoggerFactory.getLogger(PreProcessorUtils.class); @@ -39,6 +42,7 @@ public class PreProcessorUtils { public static final String CATEGORY_CHILDREN = "childrenCategories"; public static final String GLOSSARY_TERM_REL_TYPE = "AtlasGlossaryTermAnchor"; public static final String GLOSSARY_CATEGORY_REL_TYPE = "AtlasGlossaryCategoryAnchor"; + public static final String INIT_LEXORANK_OFFSET = "0|100000:"; //DataMesh models constants public static final String PARENT_DOMAIN_REL_TYPE = "parentDomain"; @@ -198,4 +202,118 @@ public static void verifyDuplicateAssetByName(String typeName, String assetName, throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, errorMessage); } } + + public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String glossaryQualifiedName, String parentQualifiedName, EntityDiscoveryService discovery) { + Map lexoRankCache = RequestContext.get().getLexoRankCache(); + + if(Objects.isNull(lexoRankCache)) { + lexoRankCache = new HashMap<>(); + } + String lexoRank = ""; + String lastLexoRank = ""; + + if(lexoRankCache.containsKey(glossaryQualifiedName + "-" + parentQualifiedName)) { + lastLexoRank = lexoRankCache.get(glossaryQualifiedName + "-" + parentQualifiedName); + + } else { + Set attributes = new HashSet<>(); + attributes.add(LEXICOGRAPHICAL_SORT_ORDER); + List categories = null; + Map dslQuery = generateDSLQueryForLastCategory(glossaryQualifiedName, parentQualifiedName); + try { + IndexSearchParams searchParams = new IndexSearchParams(); + searchParams.setAttributes(attributes); + searchParams.setDsl(dslQuery); + categories = discovery.directIndexSearch(searchParams).getEntities(); + } catch (AtlasBaseException e) { + e.printStackTrace(); + } + + if (CollectionUtils.isNotEmpty(categories)) { + for (AtlasEntityHeader category : categories) { + String lexicographicalSortOrder = (String) category.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); + if (StringUtils.isNotEmpty(lexicographicalSortOrder)) { + lastLexoRank = lexicographicalSortOrder; + } else { + lastLexoRank = INIT_LEXORANK_OFFSET; + } + } + } else { + lastLexoRank = INIT_LEXORANK_OFFSET; + } + } + + LexoRank parsedLexoRank = LexoRank.parse(lastLexoRank); + LexoRank nextLexoRank = parsedLexoRank.genNext().genNext(); + lexoRank = nextLexoRank.toString(); + + entity.setAttribute(LEXICOGRAPHICAL_SORT_ORDER, lexoRank); + lexoRankCache.put(glossaryQualifiedName + "-" + parentQualifiedName, lexoRank); + RequestContext.get().setLexoRankCache(lexoRankCache); + } + + + public static Map generateDSLQueryForLastCategory(String glossaryQualifiedName, String parentQualifiedName) { + + Map sortKeyOrder = mapOf(LEXICOGRAPHICAL_SORT_ORDER, mapOf("order", "desc")); + Map scoreSortOrder = mapOf("_score", mapOf("order", "desc")); + Map displayNameSortOrder = mapOf("displayName.keyword", mapOf("order", "desc")); + + Object[] sortArray = {sortKeyOrder, scoreSortOrder, displayNameSortOrder}; + + Map functionScore = mapOf("query", buildBoolQuery(glossaryQualifiedName, parentQualifiedName)); + + Map dsl = new HashMap<>(); + dsl.put("from", 0); + dsl.put("size", 1); + dsl.put("sort", sortArray); + dsl.put("query", mapOf("function_score", functionScore)); + + return dsl; + } + + private static Map buildBoolQuery(String glossaryQualifiedName, String parentQualifiedName) { + Map boolQuery = new HashMap<>(); + int mustArrayLength = 0; + if(StringUtils.isEmpty(parentQualifiedName) && StringUtils.isEmpty(glossaryQualifiedName)){ + mustArrayLength = 2; + } else if(StringUtils.isEmpty(parentQualifiedName) && StringUtils.isNotEmpty(glossaryQualifiedName)){ + mustArrayLength = 3; + } else { + mustArrayLength = 4; + } + Map[] mustArray = new Map[mustArrayLength]; + Map boolFilter = new HashMap<>(); + Map[] mustNotArray = new Map[2]; + + mustArray[0] = mapOf("term", mapOf("__state", "ACTIVE")); + if(StringUtils.isNotEmpty(glossaryQualifiedName)) { + mustArray[1] = mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossaryTerm", "AtlasGlossaryCategory"))); + mustArray[2] = mapOf("term", mapOf("__glossary", glossaryQualifiedName)); + } else{ + mustArray[1] = mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossary"))); + } + + if(StringUtils.isEmpty(parentQualifiedName)) { + mustNotArray[0] = mapOf("exists", mapOf("field", "__categories")); + mustNotArray[1] = mapOf("exists", mapOf("field", "__parentCategory")); + boolFilter.put("must_not", mustNotArray); + } + else { + Map[] shouldParentArray = new Map[2]; + shouldParentArray[0] = mapOf("term", mapOf("__categories", parentQualifiedName)); + shouldParentArray[1] = mapOf("term", mapOf("__parentCategory", parentQualifiedName)); + mustArray[3] = mapOf("bool",mapOf("should", shouldParentArray)); + } + + boolFilter.put("must", mustArray); + + Map nestedBoolQuery = mapOf("bool", boolFilter); + + Map topBoolFilter = mapOf("filter", nestedBoolQuery); + + boolQuery.put("bool", topBoolFilter); + + return boolQuery; + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java index 1d263ed8d8..383273d73a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java @@ -67,7 +67,6 @@ public abstract class AbstractGlossaryPreProcessor implements PreProcessor { protected static final String ATTR_MEANINGS = "meanings"; protected static final String ATTR_CATEGORIES = "categories"; - protected static final String INIT_LEXORANK_OFFSET = "0|100000:"; protected final AtlasTypeRegistry typeRegistry; protected final EntityGraphRetriever entityRetriever; @@ -247,103 +246,4 @@ protected void recordUpdatedChildEntities(AtlasVertex entityVertex, Map attributes = new HashSet<>(); - attributes.add(LEXICOGRAPHICAL_SORT_ORDER); - List categories = null; - Map dslQuery = generateDSLQueryForLastCategory(glossaryQualifiedName, parentQualifiedName); - String lexoRank = ""; - try { - categories = indexSearchPaginated(dslQuery, attributes, this.discovery); - } catch (AtlasBaseException e) { - throw new RuntimeException(e); - } - - if (CollectionUtils.isNotEmpty(categories)) { - for (AtlasEntityHeader category : categories) { - String lexicographicalSortOrder = (String) category.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); - if(StringUtils.isNotEmpty(lexicographicalSortOrder)){ - LexoRank parsedLexoRank = LexoRank.parse(lexicographicalSortOrder); - LexoRank nextLexoRank = parsedLexoRank.genNext().genNext(); - lexoRank = nextLexoRank.toString(); - } else { - LexoRank parsedLexoRank = LexoRank.parse(INIT_LEXORANK_OFFSET); - LexoRank nextLexoRank = parsedLexoRank.genNext().genNext(); - lexoRank = nextLexoRank.toString(); - } - } - } else { - LexoRank parsedLexoRank = LexoRank.parse(INIT_LEXORANK_OFFSET); - LexoRank nextLexoRank = parsedLexoRank.genNext().genNext(); - lexoRank = nextLexoRank.toString(); - } - entity.setAttribute(LEXICOGRAPHICAL_SORT_ORDER, lexoRank); - } - - - public static Map generateDSLQueryForLastCategory(String glossaryQualifiedName, String parentQualifiedName) { - - Map sortKeyOrder = mapOf(LEXICOGRAPHICAL_SORT_ORDER, mapOf("order", "desc")); - Map scoreSortOrder = mapOf("_score", mapOf("order", "desc")); - Map displayNameSortOrder = mapOf("displayName.keyword", mapOf("order", "desc")); - - Object[] sortArray = {sortKeyOrder, scoreSortOrder, displayNameSortOrder}; - - Map functionScore = mapOf("query", buildBoolQuery(glossaryQualifiedName, parentQualifiedName)); - - Map dsl = new HashMap<>(); - dsl.put("from", 0); - dsl.put("size", 1); - dsl.put("sort", sortArray); - dsl.put("query", mapOf("function_score", functionScore)); - - return dsl; - } - - private static Map buildBoolQuery(String glossaryQualifiedName, String parentQualifiedName) { - Map boolQuery = new HashMap<>(); - int mustArrayLength = 0; - if(StringUtils.isEmpty(parentQualifiedName) && StringUtils.isEmpty(glossaryQualifiedName)){ - mustArrayLength = 2; - } else if(StringUtils.isEmpty(parentQualifiedName) && StringUtils.isNotEmpty(glossaryQualifiedName)){ - mustArrayLength = 3; - } else { - mustArrayLength = 4; - } - Map[] mustArray = new Map[mustArrayLength]; - Map boolFilter = new HashMap<>(); - Map[] mustNotArray = new Map[2]; - - mustArray[0] = mapOf("term", mapOf("__state", "ACTIVE")); - if(StringUtils.isNotEmpty(glossaryQualifiedName)) { - mustArray[1] = mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossaryTerm", "AtlasGlossaryCategory"))); - mustArray[2] = mapOf("term", mapOf("__glossary", glossaryQualifiedName)); - } else{ - mustArray[1] = mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossary"))); - } - - if(StringUtils.isEmpty(parentQualifiedName)) { - mustNotArray[0] = mapOf("exists", mapOf("field", "__categories")); - mustNotArray[1] = mapOf("exists", mapOf("field", "__parentCategory")); - boolFilter.put("must_not", mustNotArray); - } - else { - Map[] shouldParentArray = new Map[2]; - shouldParentArray[0] = mapOf("term", mapOf("__categories", parentQualifiedName)); - shouldParentArray[1] = mapOf("term", mapOf("__parentCategory", parentQualifiedName)); - mustArray[3] = mapOf("bool",mapOf("should", shouldParentArray)); - } - - boolFilter.put("must", mustArray); - - Map nestedBoolQuery = mapOf("bool", boolFilter); - - Map topBoolFilter = mapOf("filter", nestedBoolQuery); - - boolQuery.put("bool", topBoolFilter); - - return boolQuery; - } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java index 532538aa1b..1ad8752fff 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java @@ -124,15 +124,16 @@ private void processCreateCategory(AtlasEntity entity, AtlasVertex vertex) throw } String glossaryQualifiedName = (String) anchor.getAttribute(QUALIFIED_NAME); + categoryExists(catName, glossaryQualifiedName); + validateParent(glossaryQualifiedName); + if (parentCategory != null) { parentQname = (String) parentCategory.getAttribute(QUALIFIED_NAME); } String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); if(StringUtils.isEmpty(lexicographicalSortOrder)){ - assignNewLexicographicalSortOrder(entity,glossaryQualifiedName, parentQname); + assignNewLexicographicalSortOrder(entity,glossaryQualifiedName, parentQname, this.discovery); } - categoryExists(catName, glossaryQualifiedName); - validateParent(glossaryQualifiedName); entity.setAttribute(QUALIFIED_NAME, createQualifiedName(vertex)); AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, new AtlasEntityHeader(entity)), diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java index 5cea0d0683..fe629d45ce 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java @@ -20,10 +20,12 @@ import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.RequestContext; +import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasStruct; import org.apache.atlas.model.instance.EntityMutations; +import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; @@ -37,8 +39,7 @@ import static org.apache.atlas.repository.Constants.NAME; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; -import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.getUUID; -import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.isNameInvalid; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.*; import static org.apache.atlas.type.Constants.LEXICOGRAPHICAL_SORT_ORDER; public class GlossaryPreProcessor implements PreProcessor { @@ -46,10 +47,16 @@ public class GlossaryPreProcessor implements PreProcessor { private final AtlasTypeRegistry typeRegistry; private final EntityGraphRetriever entityRetriever; + protected EntityDiscoveryService discovery; - public GlossaryPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever) { + public GlossaryPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, AtlasGraph graph) { this.entityRetriever = entityRetriever; this.typeRegistry = typeRegistry; + try{ + this.discovery = new EntityDiscoveryService(typeRegistry, graph, null, null, null, null); + } catch (Exception e) { + e.printStackTrace(); + } } @Override @@ -79,14 +86,16 @@ private void processCreateGlossary(AtlasStruct entity) throws AtlasBaseException throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); } String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); - // TODO : Figure the placement for this method for glossary creation. -// if(StringUtils.isEmpty(lexicographicalSortOrder)){ -// assignNewLexicographicalSortOrder(entity,null, null); -// } + + if (glossaryExists(glossaryName)) { throw new AtlasBaseException(AtlasErrorCode.GLOSSARY_ALREADY_EXISTS,glossaryName); } + if(StringUtils.isEmpty(lexicographicalSortOrder)){ + assignNewLexicographicalSortOrder((AtlasEntity) entity, null, null, this.discovery); + } + entity.setAttribute(QUALIFIED_NAME, createQualifiedName()); RequestContext.get().endMetricRecord(metricRecorder); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java index 53e12ea93e..e80c739e79 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java @@ -45,11 +45,13 @@ import org.springframework.stereotype.Component; import java.util.Iterator; import java.util.List; +import java.util.Objects; import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.graph.GraphHelper.getActiveParentVertices; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.*; import static org.apache.atlas.repository.store.graph.v2.tasks.MeaningsTaskFactory.UPDATE_ENTITY_MEANINGS_ON_TERM_UPDATE; +import static org.apache.atlas.type.Constants.LEXICOGRAPHICAL_SORT_ORDER; @Component public class TermPreProcessor extends AbstractGlossaryPreProcessor { @@ -95,7 +97,12 @@ private void processCreateTerm(AtlasEntity entity, AtlasVertex vertex) throws At termExists(termName, glossaryQName); - validateCategory(entity); + String parentQname = validateAndGetCategory(entity); + + String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); + if(StringUtils.isEmpty(lexicographicalSortOrder)){ + assignNewLexicographicalSortOrder(entity, glossaryQName, parentQname, this.discovery); + } entity.setAttribute(QUALIFIED_NAME, createQualifiedName()); AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, new AtlasEntityHeader(entity)), @@ -114,7 +121,7 @@ private void processUpdateTerm(AtlasEntity entity, AtlasVertex vertex) throws At throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); } - validateCategory(entity); + validateAndGetCategory(entity); AtlasEntity storedTerm = entityRetriever.toAtlasEntity(vertex); AtlasRelatedObjectId currentGlossary = (AtlasRelatedObjectId) storedTerm.getRelationshipAttribute(ANCHOR); @@ -159,15 +166,16 @@ private void processUpdateTerm(AtlasEntity entity, AtlasVertex vertex) throws At RequestContext.get().endMetricRecord(metricRecorder); } - private void validateCategory(AtlasEntity entity) throws AtlasBaseException { + private String validateAndGetCategory(AtlasEntity entity) throws AtlasBaseException { String glossaryQualifiedName = (String) anchor.getAttribute(QUALIFIED_NAME); + String categoryQualifiedName = null; + if (entity.hasRelationshipAttribute(ATTR_CATEGORIES) && entity.getRelationshipAttribute(ATTR_CATEGORIES) != null) { List categories = (List) entity.getRelationshipAttribute(ATTR_CATEGORIES); if (CollectionUtils.isNotEmpty(categories)) { AtlasObjectId category = categories.get(0); - String categoryQualifiedName; if (category.getUniqueAttributes() != null && category.getUniqueAttributes().containsKey(QUALIFIED_NAME)) { categoryQualifiedName = (String) category.getUniqueAttributes().get(QUALIFIED_NAME); @@ -181,6 +189,7 @@ private void validateCategory(AtlasEntity entity) throws AtlasBaseException { } } } + return categoryQualifiedName; } public String moveTermToAnotherGlossary(AtlasEntity entity, AtlasVertex vertex, diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java b/server-api/src/main/java/org/apache/atlas/RequestContext.java index 565832b7bd..3680d97a08 100644 --- a/server-api/src/main/java/org/apache/atlas/RequestContext.java +++ b/server-api/src/main/java/org/apache/atlas/RequestContext.java @@ -47,6 +47,9 @@ public class RequestContext { private final Map updatedEntities = new HashMap<>(); private final Map deletedEntities = new HashMap<>(); private final Map restoreEntities = new HashMap<>(); + + + private Map lexoRankCache = null; private final Map entityCache = new HashMap<>(); private final Map entityHeaderCache = new HashMap<>(); private final Map entityExtInfoCache = new HashMap<>(); @@ -162,6 +165,7 @@ public void clearCache() { this.requestContextHeaders.clear(); this.relationshipEndToVertexIdMap.clear(); this.relationshipMutationMap.clear(); + this.lexoRankCache = null; this.currentTask = null; this.skipAuthorizationCheck = false; this.delayTagNotifications = false; @@ -788,4 +792,12 @@ public void clearMutationContext(String event) { public Map> getRelationshipMutationMap() { return relationshipMutationMap; } + + public Map getLexoRankCache() { + return lexoRankCache; + } + + public void setLexoRankCache(Map lexoRankCache) { + this.lexoRankCache = lexoRankCache; + } } \ No newline at end of file From 360b225657b8d6637d4e28112a2a3e8ea611a23e Mon Sep 17 00:00:00 2001 From: hr2904 Date: Tue, 28 May 2024 12:48:31 +0530 Subject: [PATCH 03/13] Added LexoRank Validation method, with tentative rebalancing trigger logic --- .../v2/preprocessor/PreProcessorUtils.java | 22 +++++++++++++++++++ .../glossary/CategoryPreProcessor.java | 7 ++++++ .../glossary/GlossaryPreProcessor.java | 8 ++++++- .../glossary/TermPreProcessor.java | 8 +++++++ 4 files changed, 44 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index 53380100cb..57a1839845 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -23,6 +23,8 @@ import org.slf4j.LoggerFactory; import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import static org.apache.atlas.repository.Constants.QUERY_COLLECTION_ENTITY_TYPE; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; @@ -85,6 +87,9 @@ public enum MigrationStatus { public static final String CHILDREN_QUERIES = "__Namespace.childrenQueries"; public static final String CHILDREN_FOLDERS = "__Namespace.childrenFolders"; + public static final int REBALANCING_TRIGGER = 119; + public static final int PRE_DELIMITER_LENGTH = 9; + public static final String LEXORANK_HARD_LIMIT = "" + (256 - PRE_DELIMITER_LENGTH); public static String getUUID(){ return NanoIdUtils.randomNanoId(); @@ -203,6 +208,23 @@ public static void verifyDuplicateAssetByName(String typeName, String assetName, } } + public static void isValidLexoRank(String input) throws AtlasBaseException { + String pattern = "^0\\|[0-9a-z]{6}:(?:[0-9a-z]{0," + LEXORANK_HARD_LIMIT + "})?$"; + + Pattern regex = Pattern.compile(pattern); + + Matcher matcher = regex.matcher(input); + + if(!matcher.matches()){ + throw new AtlasBaseException("Invalid LexicographicSortOrder"); + } + // TODO : Add the rebalancing logic here + int colonIndex = input.indexOf(":"); + if (colonIndex != -1 && input.substring(colonIndex + 1).length() >= REBALANCING_TRIGGER) { + // Rebalancing trigger + } + } + public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String glossaryQualifiedName, String parentQualifiedName, EntityDiscoveryService discovery) { Map lexoRankCache = RequestContext.get().getLexoRankCache(); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java index 1ad8752fff..de11636a82 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java @@ -133,6 +133,8 @@ private void processCreateCategory(AtlasEntity entity, AtlasVertex vertex) throw String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); if(StringUtils.isEmpty(lexicographicalSortOrder)){ assignNewLexicographicalSortOrder(entity,glossaryQualifiedName, parentQname, this.discovery); + } else { + isValidLexoRank(lexicographicalSortOrder); } entity.setAttribute(QUALIFIED_NAME, createQualifiedName(vertex)); @@ -160,6 +162,11 @@ private void processUpdateCategory(AtlasEntity entity, AtlasVertex vertex) throw String newGlossaryQualifiedName = (String) anchor.getAttribute(QUALIFIED_NAME); + String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); + if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { + isValidLexoRank(lexicographicalSortOrder); + } + if (!currentGlossaryQualifiedName.equals(newGlossaryQualifiedName)){ //Auth check isAuthorized(currentGlossaryHeader, anchor); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java index fe629d45ce..d74bc54528 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java @@ -92,8 +92,10 @@ private void processCreateGlossary(AtlasStruct entity) throws AtlasBaseException throw new AtlasBaseException(AtlasErrorCode.GLOSSARY_ALREADY_EXISTS,glossaryName); } - if(StringUtils.isEmpty(lexicographicalSortOrder)){ + if(StringUtils.isEmpty(lexicographicalSortOrder)) { assignNewLexicographicalSortOrder((AtlasEntity) entity, null, null, this.discovery); + } else { + isValidLexoRank(lexicographicalSortOrder); } entity.setAttribute(QUALIFIED_NAME, createQualifiedName()); @@ -112,6 +114,10 @@ private void processUpdateGlossary(AtlasStruct entity, AtlasVertex vertex) throw if (StringUtils.isEmpty(glossaryName) || isNameInvalid(glossaryName)) { throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); } + String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); + if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { + isValidLexoRank(lexicographicalSortOrder); + } String vertexQnName = vertex.getProperty(QUALIFIED_NAME, String.class); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java index e80c739e79..36528f0fe7 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java @@ -95,6 +95,7 @@ private void processCreateTerm(AtlasEntity entity, AtlasVertex vertex) throws At String glossaryQName = (String) anchor.getAttribute(QUALIFIED_NAME); + termExists(termName, glossaryQName); String parentQname = validateAndGetCategory(entity); @@ -102,6 +103,8 @@ private void processCreateTerm(AtlasEntity entity, AtlasVertex vertex) throws At String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); if(StringUtils.isEmpty(lexicographicalSortOrder)){ assignNewLexicographicalSortOrder(entity, glossaryQName, parentQname, this.discovery); + } else { + isValidLexoRank(lexicographicalSortOrder); } entity.setAttribute(QUALIFIED_NAME, createQualifiedName()); @@ -132,6 +135,11 @@ private void processUpdateTerm(AtlasEntity entity, AtlasVertex vertex) throws At String newGlossaryQualifiedName = (String) anchor.getAttribute(QUALIFIED_NAME); + String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); + if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { + isValidLexoRank(lexicographicalSortOrder); + } + if (!currentGlossaryQualifiedName.equals(newGlossaryQualifiedName)){ //Auth check isAuthorized(currentGlossaryHeader, anchor); From 6a55f1338b4924dd750d81cfb5963ef2a6a4b808 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Tue, 28 May 2024 18:00:28 +0530 Subject: [PATCH 04/13] Corrected a method rename --- .../store/graph/v2/preprocessor/glossary/TermPreProcessor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java index 36528f0fe7..39e8654437 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java @@ -176,7 +176,6 @@ private void processUpdateTerm(AtlasEntity entity, AtlasVertex vertex) throws At private String validateAndGetCategory(AtlasEntity entity) throws AtlasBaseException { String glossaryQualifiedName = (String) anchor.getAttribute(QUALIFIED_NAME); - String categoryQualifiedName = null; if (entity.hasRelationshipAttribute(ATTR_CATEGORIES) && entity.getRelationshipAttribute(ATTR_CATEGORIES) != null) { From 0de8f9542fe34829971bf835016435eb59d7d874 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Wed, 29 May 2024 01:05:33 +0530 Subject: [PATCH 05/13] Added a check in validateLexoRank which checks if another entity with duplicate lexorank doesnt exist in scenario where lexorank is being sent from user. --- .../v2/preprocessor/PreProcessorUtils.java | 80 ++++++++++++++++++- .../glossary/CategoryPreProcessor.java | 5 +- .../glossary/GlossaryPreProcessor.java | 4 +- .../glossary/TermPreProcessor.java | 6 +- 4 files changed, 87 insertions(+), 8 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index 57a1839845..36547c7c87 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -208,7 +208,7 @@ public static void verifyDuplicateAssetByName(String typeName, String assetName, } } - public static void isValidLexoRank(String input) throws AtlasBaseException { + public static void isValidLexoRank(String input, String glossaryQualifiedName, String parentQualifiedName, EntityDiscoveryService discovery) throws AtlasBaseException { String pattern = "^0\\|[0-9a-z]{6}:(?:[0-9a-z]{0," + LEXORANK_HARD_LIMIT + "})?$"; Pattern regex = Pattern.compile(pattern); @@ -218,6 +218,21 @@ public static void isValidLexoRank(String input) throws AtlasBaseException { if(!matcher.matches()){ throw new AtlasBaseException("Invalid LexicographicSortOrder"); } + + Map dslQuery = createDSLforCheckingPreExistingLexoRank(input, glossaryQualifiedName, parentQualifiedName); + List categories = new ArrayList<>(); + try { + IndexSearchParams searchParams = new IndexSearchParams(); + searchParams.setAttributes(new HashSet<>()); + searchParams.setDsl(dslQuery); + categories = discovery.directIndexSearch(searchParams).getEntities(); + } catch (AtlasBaseException e) { + e.printStackTrace(); + } + + if(!CollectionUtils.isEmpty(categories)){ + throw new AtlasBaseException("Invalid LexicographicSortOrder"); + } // TODO : Add the rebalancing logic here int colonIndex = input.indexOf(":"); if (colonIndex != -1 && input.substring(colonIndex + 1).length() >= REBALANCING_TRIGGER) { @@ -274,7 +289,70 @@ public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String RequestContext.get().setLexoRankCache(lexoRankCache); } + public static Map createDSLforCheckingPreExistingLexoRank(String lexoRank, String glossaryQualifiedName, String parentQualifiedName) { + + Map sortKeyOrder = mapOf(LEXICOGRAPHICAL_SORT_ORDER, mapOf("order", "desc")); + Map scoreSortOrder = mapOf("_score", mapOf("order", "desc")); + Map displayNameSortOrder = mapOf("displayName.keyword", mapOf("order", "desc")); + + Object[] sortArray = {sortKeyOrder, scoreSortOrder, displayNameSortOrder}; + + Map functionScore = mapOf("query", buildBoolQueryDuplicateLexoRank(lexoRank, glossaryQualifiedName, parentQualifiedName)); + + Map dsl = new HashMap<>(); + dsl.put("from", 0); + dsl.put("size", 100); + dsl.put("sort", sortArray); + dsl.put("query", mapOf("function_score", functionScore)); + + return dsl; + } + + private static Map buildBoolQueryDuplicateLexoRank(String lexoRank, String glossaryQualifiedName, String parentQualifiedName) { + Map boolQuery = new HashMap<>(); + int mustArrayLength = 0; + if(StringUtils.isEmpty(parentQualifiedName) && StringUtils.isEmpty(glossaryQualifiedName)){ + mustArrayLength = 3; + } else if(StringUtils.isEmpty(parentQualifiedName) && StringUtils.isNotEmpty(glossaryQualifiedName)){ + mustArrayLength = 4; + } else { + mustArrayLength = 5; + } + Map[] mustArray = new Map[mustArrayLength]; + Map boolFilter = new HashMap<>(); + Map[] mustNotArray = new Map[2]; + + mustArray[0] = mapOf("term", mapOf("__state", "ACTIVE")); + mustArray[1] = mapOf("term", mapOf(LEXICOGRAPHICAL_SORT_ORDER, lexoRank)); + if(StringUtils.isNotEmpty(glossaryQualifiedName)) { + mustArray[2] = mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossaryTerm", "AtlasGlossaryCategory"))); + mustArray[3] = mapOf("term", mapOf("__glossary", glossaryQualifiedName)); + } else{ + mustArray[2] = mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossary"))); + } + if(StringUtils.isEmpty(parentQualifiedName)) { + mustNotArray[0] = mapOf("exists", mapOf("field", "__categories")); + mustNotArray[1] = mapOf("exists", mapOf("field", "__parentCategory")); + boolFilter.put("must_not", mustNotArray); + } + else { + Map[] shouldParentArray = new Map[2]; + shouldParentArray[0] = mapOf("term", mapOf("__categories", parentQualifiedName)); + shouldParentArray[1] = mapOf("term", mapOf("__parentCategory", parentQualifiedName)); + mustArray[4] = mapOf("bool",mapOf("should", shouldParentArray)); + } + + boolFilter.put("must", mustArray); + + Map nestedBoolQuery = mapOf("bool", boolFilter); + + Map topBoolFilter = mapOf("filter", nestedBoolQuery); + + boolQuery.put("bool", topBoolFilter); + + return boolQuery; + } public static Map generateDSLQueryForLastCategory(String glossaryQualifiedName, String parentQualifiedName) { Map sortKeyOrder = mapOf(LEXICOGRAPHICAL_SORT_ORDER, mapOf("order", "desc")); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java index de11636a82..0f90531561 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java @@ -134,7 +134,7 @@ private void processCreateCategory(AtlasEntity entity, AtlasVertex vertex) throw if(StringUtils.isEmpty(lexicographicalSortOrder)){ assignNewLexicographicalSortOrder(entity,glossaryQualifiedName, parentQname, this.discovery); } else { - isValidLexoRank(lexicographicalSortOrder); + isValidLexoRank(lexicographicalSortOrder, glossaryQualifiedName, parentQname, this.discovery); } entity.setAttribute(QUALIFIED_NAME, createQualifiedName(vertex)); @@ -163,8 +163,9 @@ private void processUpdateCategory(AtlasEntity entity, AtlasVertex vertex) throw String newGlossaryQualifiedName = (String) anchor.getAttribute(QUALIFIED_NAME); String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); + String parentQname = (String) parentCategory.getAttribute(QUALIFIED_NAME); if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { - isValidLexoRank(lexicographicalSortOrder); + isValidLexoRank(lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery); } if (!currentGlossaryQualifiedName.equals(newGlossaryQualifiedName)){ diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java index d74bc54528..b585705284 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java @@ -95,7 +95,7 @@ private void processCreateGlossary(AtlasStruct entity) throws AtlasBaseException if(StringUtils.isEmpty(lexicographicalSortOrder)) { assignNewLexicographicalSortOrder((AtlasEntity) entity, null, null, this.discovery); } else { - isValidLexoRank(lexicographicalSortOrder); + isValidLexoRank(lexicographicalSortOrder, "", "", this.discovery); } entity.setAttribute(QUALIFIED_NAME, createQualifiedName()); @@ -116,7 +116,7 @@ private void processUpdateGlossary(AtlasStruct entity, AtlasVertex vertex) throw } String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { - isValidLexoRank(lexicographicalSortOrder); + isValidLexoRank(lexicographicalSortOrder, "", "", this.discovery); } String vertexQnName = vertex.getProperty(QUALIFIED_NAME, String.class); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java index 39e8654437..9af03cabf8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java @@ -104,7 +104,7 @@ private void processCreateTerm(AtlasEntity entity, AtlasVertex vertex) throws At if(StringUtils.isEmpty(lexicographicalSortOrder)){ assignNewLexicographicalSortOrder(entity, glossaryQName, parentQname, this.discovery); } else { - isValidLexoRank(lexicographicalSortOrder); + isValidLexoRank(lexicographicalSortOrder, glossaryQName, parentQname, this.discovery); } entity.setAttribute(QUALIFIED_NAME, createQualifiedName()); @@ -124,7 +124,7 @@ private void processUpdateTerm(AtlasEntity entity, AtlasVertex vertex) throws At throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); } - validateAndGetCategory(entity); + String parentQname = validateAndGetCategory(entity); AtlasEntity storedTerm = entityRetriever.toAtlasEntity(vertex); AtlasRelatedObjectId currentGlossary = (AtlasRelatedObjectId) storedTerm.getRelationshipAttribute(ANCHOR); @@ -137,7 +137,7 @@ private void processUpdateTerm(AtlasEntity entity, AtlasVertex vertex) throws At String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { - isValidLexoRank(lexicographicalSortOrder); + isValidLexoRank(lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery); } if (!currentGlossaryQualifiedName.equals(newGlossaryQualifiedName)){ From 76dbc232766ad9617ae646aa05f5abde6f9ab07f Mon Sep 17 00:00:00 2001 From: hr2904 Date: Wed, 29 May 2024 17:15:35 +0530 Subject: [PATCH 06/13] Added a check , such that if bulk request is from migration, it will override the check for for duplicate lexorank. --- .../v2/preprocessor/PreProcessorUtils.java | 30 ++++++++++--------- .../glossary/CategoryPreProcessor.java | 5 +++- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index 36547c7c87..889273b65f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -210,7 +210,8 @@ public static void verifyDuplicateAssetByName(String typeName, String assetName, public static void isValidLexoRank(String input, String glossaryQualifiedName, String parentQualifiedName, EntityDiscoveryService discovery) throws AtlasBaseException { String pattern = "^0\\|[0-9a-z]{6}:(?:[0-9a-z]{0," + LEXORANK_HARD_LIMIT + "})?$"; - + // TODO : To remove this after migration is successful on all tenants and custom-sort is successfully GA + Boolean requestFromMigration = RequestContext.get().getRequestContextHeaders().getOrDefault("x-atlan-request-id", "").contains("custom-sort-migration"); Pattern regex = Pattern.compile(pattern); Matcher matcher = regex.matcher(input); @@ -218,20 +219,21 @@ public static void isValidLexoRank(String input, String glossaryQualifiedName, S if(!matcher.matches()){ throw new AtlasBaseException("Invalid LexicographicSortOrder"); } + if(!requestFromMigration) { + Map dslQuery = createDSLforCheckingPreExistingLexoRank(input, glossaryQualifiedName, parentQualifiedName); + List categories = new ArrayList<>(); + try { + IndexSearchParams searchParams = new IndexSearchParams(); + searchParams.setAttributes(new HashSet<>()); + searchParams.setDsl(dslQuery); + categories = discovery.directIndexSearch(searchParams).getEntities(); + } catch (AtlasBaseException e) { + e.printStackTrace(); + } - Map dslQuery = createDSLforCheckingPreExistingLexoRank(input, glossaryQualifiedName, parentQualifiedName); - List categories = new ArrayList<>(); - try { - IndexSearchParams searchParams = new IndexSearchParams(); - searchParams.setAttributes(new HashSet<>()); - searchParams.setDsl(dslQuery); - categories = discovery.directIndexSearch(searchParams).getEntities(); - } catch (AtlasBaseException e) { - e.printStackTrace(); - } - - if(!CollectionUtils.isEmpty(categories)){ - throw new AtlasBaseException("Invalid LexicographicSortOrder"); + if (!CollectionUtils.isEmpty(categories)) { + throw new AtlasBaseException("Invalid LexicographicSortOrder"); + } } // TODO : Add the rebalancing logic here int colonIndex = input.indexOf(":"); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java index 0f90531561..baac9c82c3 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java @@ -163,7 +163,10 @@ private void processUpdateCategory(AtlasEntity entity, AtlasVertex vertex) throw String newGlossaryQualifiedName = (String) anchor.getAttribute(QUALIFIED_NAME); String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); - String parentQname = (String) parentCategory.getAttribute(QUALIFIED_NAME); + String parentQname = ""; + if(Objects.nonNull(parentCategory)) { + parentQname = (String) parentCategory.getAttribute(QUALIFIED_NAME); + } if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { isValidLexoRank(lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery); } From 5a4f235b036ede15a6a4f30b4ea990a9b4d3a87f Mon Sep 17 00:00:00 2001 From: hr2904 Date: Mon, 3 Jun 2024 00:00:21 +0530 Subject: [PATCH 07/13] Modified ES query. --- .../org/apache/atlas/glossary/GlossaryUtils.java | 4 ++-- .../graph/v2/preprocessor/PreProcessorUtils.java | 13 ++++++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/glossary/GlossaryUtils.java b/repository/src/main/java/org/apache/atlas/glossary/GlossaryUtils.java index 0f16d0dc82..2310726b2e 100644 --- a/repository/src/main/java/org/apache/atlas/glossary/GlossaryUtils.java +++ b/repository/src/main/java/org/apache/atlas/glossary/GlossaryUtils.java @@ -55,8 +55,8 @@ public abstract class GlossaryUtils { public static final String TERM_ASSIGNMENT_ATTR_SOURCE = "source"; static final String ATLAS_GLOSSARY_TYPENAME = "AtlasGlossary"; - static final String ATLAS_GLOSSARY_TERM_TYPENAME = "AtlasGlossaryTerm"; - static final String ATLAS_GLOSSARY_CATEGORY_TYPENAME = "AtlasGlossaryCategory"; + public static final String ATLAS_GLOSSARY_TERM_TYPENAME = "AtlasGlossaryTerm"; + public static final String ATLAS_GLOSSARY_CATEGORY_TYPENAME = "AtlasGlossaryCategory"; public static final String NAME = "name"; public static final String QUALIFIED_NAME = "qualifiedName"; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index 889273b65f..287bb0fc3f 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -26,6 +26,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import static org.apache.atlas.glossary.GlossaryUtils.ATLAS_GLOSSARY_TERM_TYPENAME; import static org.apache.atlas.repository.Constants.QUERY_COLLECTION_ENTITY_TYPE; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY; @@ -258,7 +259,8 @@ public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String Set attributes = new HashSet<>(); attributes.add(LEXICOGRAPHICAL_SORT_ORDER); List categories = null; - Map dslQuery = generateDSLQueryForLastCategory(glossaryQualifiedName, parentQualifiedName); + boolean isTerm = entity.getTypeName().equals(ATLAS_GLOSSARY_TERM_TYPENAME) ? true : false; + Map dslQuery = generateDSLQueryForLastCategory(glossaryQualifiedName, parentQualifiedName, isTerm); try { IndexSearchParams searchParams = new IndexSearchParams(); searchParams.setAttributes(attributes); @@ -355,7 +357,7 @@ private static Map buildBoolQueryDuplicateLexoRank(String lexoRa return boolQuery; } - public static Map generateDSLQueryForLastCategory(String glossaryQualifiedName, String parentQualifiedName) { + public static Map generateDSLQueryForLastCategory(String glossaryQualifiedName, String parentQualifiedName, boolean isTerm) { Map sortKeyOrder = mapOf(LEXICOGRAPHICAL_SORT_ORDER, mapOf("order", "desc")); Map scoreSortOrder = mapOf("_score", mapOf("order", "desc")); @@ -363,7 +365,7 @@ public static Map generateDSLQueryForLastCategory(String glossar Object[] sortArray = {sortKeyOrder, scoreSortOrder, displayNameSortOrder}; - Map functionScore = mapOf("query", buildBoolQuery(glossaryQualifiedName, parentQualifiedName)); + Map functionScore = mapOf("query", buildBoolQuery(glossaryQualifiedName, parentQualifiedName, isTerm)); Map dsl = new HashMap<>(); dsl.put("from", 0); @@ -374,7 +376,7 @@ public static Map generateDSLQueryForLastCategory(String glossar return dsl; } - private static Map buildBoolQuery(String glossaryQualifiedName, String parentQualifiedName) { + private static Map buildBoolQuery(String glossaryQualifiedName, String parentQualifiedName, boolean isTerm) { Map boolQuery = new HashMap<>(); int mustArrayLength = 0; if(StringUtils.isEmpty(parentQualifiedName) && StringUtils.isEmpty(glossaryQualifiedName)){ @@ -390,7 +392,8 @@ private static Map buildBoolQuery(String glossaryQualifiedName, mustArray[0] = mapOf("term", mapOf("__state", "ACTIVE")); if(StringUtils.isNotEmpty(glossaryQualifiedName)) { - mustArray[1] = mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossaryTerm", "AtlasGlossaryCategory"))); + String typeName = isTerm ? "AtlasGlossaryTerm" : "AtlasGlossaryCategory"; + mustArray[1] = mapOf("term", mapOf("__typeName.keyword", typeName)); mustArray[2] = mapOf("term", mapOf("__glossary", glossaryQualifiedName)); } else{ mustArray[1] = mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossary"))); From d8858f06cbf000527d58e312cf989d67d62b6973 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Mon, 3 Jun 2024 11:53:17 +0530 Subject: [PATCH 08/13] Modified an edge statement such that, when a new lexorank for cat is generated, it is not a duplicate of a term in same linear data set --- .../graph/v2/preprocessor/PreProcessorUtils.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index 287bb0fc3f..b3a1fd42ff 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -274,7 +274,7 @@ public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String for (AtlasEntityHeader category : categories) { String lexicographicalSortOrder = (String) category.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); if (StringUtils.isNotEmpty(lexicographicalSortOrder)) { - lastLexoRank = lexicographicalSortOrder; + lastLexoRank = getValidLexorank(lexicographicalSortOrder, glossaryQualifiedName, parentQualifiedName, discovery); } else { lastLexoRank = INIT_LEXORANK_OFFSET; } @@ -293,6 +293,17 @@ public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String RequestContext.get().setLexoRankCache(lexoRankCache); } + private static String getValidLexorank(String lastLexoRank, String glossaryQualifiedName, String parentQualifiedName, EntityDiscoveryService discovery) { + LexoRank parsedLexoRank = LexoRank.parse(lastLexoRank); + LexoRank nextLexoRank = parsedLexoRank.genNext().genNext(); + try { + isValidLexoRank(nextLexoRank.toString(), glossaryQualifiedName, parentQualifiedName, discovery); + return lastLexoRank; + } catch (AtlasBaseException e){ + return parsedLexoRank.between(nextLexoRank).genPrev().genPrev().toString(); + } + } + public static Map createDSLforCheckingPreExistingLexoRank(String lexoRank, String glossaryQualifiedName, String parentQualifiedName) { Map sortKeyOrder = mapOf(LEXICOGRAPHICAL_SORT_ORDER, mapOf("order", "desc")); From 41622bf44bf0ae796a7fda91edb61f7500f7485e Mon Sep 17 00:00:00 2001 From: hr2904 Date: Mon, 3 Jun 2024 12:53:35 +0530 Subject: [PATCH 09/13] Reverted the check, instead added padded new offset for terms. --- .../v2/preprocessor/PreProcessorUtils.java | 56 +++++++------------ 1 file changed, 20 insertions(+), 36 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index b3a1fd42ff..d2e30931fd 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -17,6 +17,7 @@ import org.apache.atlas.util.NanoIdUtils; import org.apache.atlas.util.lexoRank.LexoRank; import org.apache.atlas.utils.AtlasEntityUtil; +import org.apache.atlas.v1.model.instance.Id; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; @@ -30,6 +31,8 @@ import static org.apache.atlas.repository.Constants.QUERY_COLLECTION_ENTITY_TYPE; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_IS_ENABLED; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_SERVICE_NAME; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; import static org.apache.atlas.type.Constants.LEXICOGRAPHICAL_SORT_ORDER; @@ -46,6 +49,7 @@ public class PreProcessorUtils { public static final String GLOSSARY_TERM_REL_TYPE = "AtlasGlossaryTermAnchor"; public static final String GLOSSARY_CATEGORY_REL_TYPE = "AtlasGlossaryCategoryAnchor"; public static final String INIT_LEXORANK_OFFSET = "0|100000:"; + public static final String INIT_TERM_LEXORANK_OFFSET = "0|500000:"; //DataMesh models constants public static final String PARENT_DOMAIN_REL_TYPE = "parentDomain"; @@ -274,13 +278,13 @@ public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String for (AtlasEntityHeader category : categories) { String lexicographicalSortOrder = (String) category.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); if (StringUtils.isNotEmpty(lexicographicalSortOrder)) { - lastLexoRank = getValidLexorank(lexicographicalSortOrder, glossaryQualifiedName, parentQualifiedName, discovery); + lastLexoRank = lexicographicalSortOrder; } else { - lastLexoRank = INIT_LEXORANK_OFFSET; + lastLexoRank = isTerm ? INIT_TERM_LEXORANK_OFFSET : INIT_LEXORANK_OFFSET; } } } else { - lastLexoRank = INIT_LEXORANK_OFFSET; + lastLexoRank = isTerm ? INIT_TERM_LEXORANK_OFFSET : INIT_LEXORANK_OFFSET; } } @@ -293,17 +297,6 @@ public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String RequestContext.get().setLexoRankCache(lexoRankCache); } - private static String getValidLexorank(String lastLexoRank, String glossaryQualifiedName, String parentQualifiedName, EntityDiscoveryService discovery) { - LexoRank parsedLexoRank = LexoRank.parse(lastLexoRank); - LexoRank nextLexoRank = parsedLexoRank.genNext().genNext(); - try { - isValidLexoRank(nextLexoRank.toString(), glossaryQualifiedName, parentQualifiedName, discovery); - return lastLexoRank; - } catch (AtlasBaseException e){ - return parsedLexoRank.between(nextLexoRank).genPrev().genPrev().toString(); - } - } - public static Map createDSLforCheckingPreExistingLexoRank(String lexoRank, String glossaryQualifiedName, String parentQualifiedName) { Map sortKeyOrder = mapOf(LEXICOGRAPHICAL_SORT_ORDER, mapOf("order", "desc")); @@ -325,37 +318,28 @@ public static Map createDSLforCheckingPreExistingLexoRank(String private static Map buildBoolQueryDuplicateLexoRank(String lexoRank, String glossaryQualifiedName, String parentQualifiedName) { Map boolQuery = new HashMap<>(); - int mustArrayLength = 0; - if(StringUtils.isEmpty(parentQualifiedName) && StringUtils.isEmpty(glossaryQualifiedName)){ - mustArrayLength = 3; - } else if(StringUtils.isEmpty(parentQualifiedName) && StringUtils.isNotEmpty(glossaryQualifiedName)){ - mustArrayLength = 4; - } else { - mustArrayLength = 5; - } - Map[] mustArray = new Map[mustArrayLength]; Map boolFilter = new HashMap<>(); - Map[] mustNotArray = new Map[2]; - - mustArray[0] = mapOf("term", mapOf("__state", "ACTIVE")); - mustArray[1] = mapOf("term", mapOf(LEXICOGRAPHICAL_SORT_ORDER, lexoRank)); + List> mustArray = new ArrayList<>(); + List> mustNotArray = new ArrayList<>(); + mustArray.add(mapOf("term", mapOf("__state", "ACTIVE"))); + mustArray.add(mapOf("term", mapOf(LEXICOGRAPHICAL_SORT_ORDER, lexoRank))); if(StringUtils.isNotEmpty(glossaryQualifiedName)) { - mustArray[2] = mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossaryTerm", "AtlasGlossaryCategory"))); - mustArray[3] = mapOf("term", mapOf("__glossary", glossaryQualifiedName)); + mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossaryTerm", "AtlasGlossaryCategory")))); + mustArray.add(mapOf("term", mapOf("__glossary", glossaryQualifiedName))); } else{ - mustArray[2] = mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossary"))); + mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossary")))); } if(StringUtils.isEmpty(parentQualifiedName)) { - mustNotArray[0] = mapOf("exists", mapOf("field", "__categories")); - mustNotArray[1] = mapOf("exists", mapOf("field", "__parentCategory")); + mustNotArray.add(mapOf("exists", mapOf("field", "__categories"))); + mustNotArray.add(mapOf("exists", mapOf("field", "__parentCategory"))); boolFilter.put("must_not", mustNotArray); } else { - Map[] shouldParentArray = new Map[2]; - shouldParentArray[0] = mapOf("term", mapOf("__categories", parentQualifiedName)); - shouldParentArray[1] = mapOf("term", mapOf("__parentCategory", parentQualifiedName)); - mustArray[4] = mapOf("bool",mapOf("should", shouldParentArray)); + List> shouldParentArray = new ArrayList<>(); + shouldParentArray.add(mapOf("term", mapOf("__categories", parentQualifiedName))); + shouldParentArray.add(mapOf("term", mapOf("__parentCategory", parentQualifiedName))); + mustArray.add(mapOf("bool",mapOf("should", shouldParentArray))); } boolFilter.put("must", mustArray); From 7b6a4abe1277c54117ff61c7fa310dc1e353aa48 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Tue, 4 Jun 2024 11:29:21 +0530 Subject: [PATCH 10/13] fixed the caching logic, by adding the bifurcation for terms and categories --- .../store/graph/v2/preprocessor/PreProcessorUtils.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index d2e30931fd..3dd3b26885 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -255,15 +255,15 @@ public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String } String lexoRank = ""; String lastLexoRank = ""; + boolean isTerm = entity.getTypeName().equals(ATLAS_GLOSSARY_TERM_TYPENAME) ? true : false; - if(lexoRankCache.containsKey(glossaryQualifiedName + "-" + parentQualifiedName)) { - lastLexoRank = lexoRankCache.get(glossaryQualifiedName + "-" + parentQualifiedName); + if(lexoRankCache.containsKey(entity.getTypeName() + "-" + glossaryQualifiedName + "-" + parentQualifiedName)) { + lastLexoRank = lexoRankCache.get(entity.getTypeName() + "-" + glossaryQualifiedName + "-" + parentQualifiedName); } else { Set attributes = new HashSet<>(); attributes.add(LEXICOGRAPHICAL_SORT_ORDER); List categories = null; - boolean isTerm = entity.getTypeName().equals(ATLAS_GLOSSARY_TERM_TYPENAME) ? true : false; Map dslQuery = generateDSLQueryForLastCategory(glossaryQualifiedName, parentQualifiedName, isTerm); try { IndexSearchParams searchParams = new IndexSearchParams(); @@ -293,7 +293,7 @@ public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String lexoRank = nextLexoRank.toString(); entity.setAttribute(LEXICOGRAPHICAL_SORT_ORDER, lexoRank); - lexoRankCache.put(glossaryQualifiedName + "-" + parentQualifiedName, lexoRank); + lexoRankCache.put(entity.getTypeName() + "-" + glossaryQualifiedName + "-" + parentQualifiedName, lexoRank); RequestContext.get().setLexoRankCache(lexoRankCache); } From 8c7b50d83a8ffbb111b677ea5d2c44a212cc09dd Mon Sep 17 00:00:00 2001 From: hr2904 Date: Wed, 12 Jun 2024 18:26:50 +0530 Subject: [PATCH 11/13] Fixed PR comments --- .../v2/preprocessor/PreProcessorUtils.java | 161 ++++++++---------- .../glossary/CategoryPreProcessor.java | 4 +- .../glossary/GlossaryPreProcessor.java | 4 +- .../glossary/TermPreProcessor.java | 4 +- .../lexoRank/system/LexoNumeralSystem10.java | 32 ---- 5 files changed, 76 insertions(+), 129 deletions(-) delete mode 100644 repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem10.java diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index 3dd3b26885..5e363035ed 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -17,7 +17,6 @@ import org.apache.atlas.util.NanoIdUtils; import org.apache.atlas.util.lexoRank.LexoRank; import org.apache.atlas.utils.AtlasEntityUtil; -import org.apache.atlas.v1.model.instance.Id; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; @@ -27,12 +26,9 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import static org.apache.atlas.glossary.GlossaryUtils.ATLAS_GLOSSARY_CATEGORY_TYPENAME; import static org.apache.atlas.glossary.GlossaryUtils.ATLAS_GLOSSARY_TERM_TYPENAME; -import static org.apache.atlas.repository.Constants.QUERY_COLLECTION_ENTITY_TYPE; -import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; -import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY; -import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_IS_ENABLED; -import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_SERVICE_NAME; +import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; import static org.apache.atlas.type.Constants.LEXICOGRAPHICAL_SORT_ORDER; @@ -95,6 +91,13 @@ public enum MigrationStatus { public static final int REBALANCING_TRIGGER = 119; public static final int PRE_DELIMITER_LENGTH = 9; public static final String LEXORANK_HARD_LIMIT = "" + (256 - PRE_DELIMITER_LENGTH); + public static final String LEXORANK_VALID_PATTERN = "^0\\|[0-9a-z]{6}:(?:[0-9a-z]{0," + LEXORANK_HARD_LIMIT + "})?$"; + public static final Set ATTRIBUTES; + static { + Set temp = new HashSet<>(); + temp.add(LEXICOGRAPHICAL_SORT_ORDER); + ATTRIBUTES = Collections.unmodifiableSet(temp); + } public static String getUUID(){ return NanoIdUtils.randomNanoId(); @@ -213,33 +216,35 @@ public static void verifyDuplicateAssetByName(String typeName, String assetName, } } - public static void isValidLexoRank(String input, String glossaryQualifiedName, String parentQualifiedName, EntityDiscoveryService discovery) throws AtlasBaseException { - String pattern = "^0\\|[0-9a-z]{6}:(?:[0-9a-z]{0," + LEXORANK_HARD_LIMIT + "})?$"; + public static void isValidLexoRank(boolean isTerm, String input, String glossaryQualifiedName, String parentQualifiedName, EntityDiscoveryService discovery) throws AtlasBaseException { // TODO : To remove this after migration is successful on all tenants and custom-sort is successfully GA - Boolean requestFromMigration = RequestContext.get().getRequestContextHeaders().getOrDefault("x-atlan-request-id", "").contains("custom-sort-migration"); - Pattern regex = Pattern.compile(pattern); + Pattern regex = Pattern.compile(LEXORANK_VALID_PATTERN); Matcher matcher = regex.matcher(input); if(!matcher.matches()){ - throw new AtlasBaseException("Invalid LexicographicSortOrder"); + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Invalid value for attribute"); + } + Boolean requestFromMigration = RequestContext.get().getRequestContextHeaders().getOrDefault("x-atlan-request-id", "").contains("custom-sort-migration"); + if(requestFromMigration) { + return; + } + Map dslQuery = createDSLforCheckingPreExistingLexoRank(isTerm, input, glossaryQualifiedName, parentQualifiedName); + List assetsWithDuplicateRank = new ArrayList<>(); + try { + IndexSearchParams searchParams = new IndexSearchParams(); + searchParams.setAttributes(new HashSet<>()); + searchParams.setDsl(dslQuery); + assetsWithDuplicateRank = discovery.directIndexSearch(searchParams).getEntities(); + } catch (AtlasBaseException e) { + LOG.error("IndexSearch Error Occured : " + e.getMessage()); + new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Something went wrong with IndexSearch"); } - if(!requestFromMigration) { - Map dslQuery = createDSLforCheckingPreExistingLexoRank(input, glossaryQualifiedName, parentQualifiedName); - List categories = new ArrayList<>(); - try { - IndexSearchParams searchParams = new IndexSearchParams(); - searchParams.setAttributes(new HashSet<>()); - searchParams.setDsl(dslQuery); - categories = discovery.directIndexSearch(searchParams).getEntities(); - } catch (AtlasBaseException e) { - e.printStackTrace(); - } - if (!CollectionUtils.isEmpty(categories)) { - throw new AtlasBaseException("Invalid LexicographicSortOrder"); - } + if (!CollectionUtils.isEmpty(assetsWithDuplicateRank)) { + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Duplicate Lexorank found"); } + // TODO : Add the rebalancing logic here int colonIndex = input.indexOf(":"); if (colonIndex != -1 && input.substring(colonIndex + 1).length() >= REBALANCING_TRIGGER) { @@ -255,19 +260,18 @@ public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String } String lexoRank = ""; String lastLexoRank = ""; - boolean isTerm = entity.getTypeName().equals(ATLAS_GLOSSARY_TERM_TYPENAME) ? true : false; - - if(lexoRankCache.containsKey(entity.getTypeName() + "-" + glossaryQualifiedName + "-" + parentQualifiedName)) { - lastLexoRank = lexoRankCache.get(entity.getTypeName() + "-" + glossaryQualifiedName + "-" + parentQualifiedName); + boolean isTerm = entity.getTypeName().equals(ATLAS_GLOSSARY_TERM_TYPENAME); + String cacheKey = entity.getTypeName() + "-" + glossaryQualifiedName + "-" + parentQualifiedName; + if(lexoRankCache.containsKey(cacheKey)) { + lastLexoRank = lexoRankCache.get(cacheKey); } else { - Set attributes = new HashSet<>(); - attributes.add(LEXICOGRAPHICAL_SORT_ORDER); + List categories = null; - Map dslQuery = generateDSLQueryForLastCategory(glossaryQualifiedName, parentQualifiedName, isTerm); + Map dslQuery = generateDSLQueryForLastChild(glossaryQualifiedName, parentQualifiedName, isTerm); try { IndexSearchParams searchParams = new IndexSearchParams(); - searchParams.setAttributes(attributes); + searchParams.setAttributes(ATTRIBUTES); searchParams.setDsl(dslQuery); categories = discovery.directIndexSearch(searchParams).getEntities(); } catch (AtlasBaseException e) { @@ -297,123 +301,98 @@ public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String RequestContext.get().setLexoRankCache(lexoRankCache); } - public static Map createDSLforCheckingPreExistingLexoRank(String lexoRank, String glossaryQualifiedName, String parentQualifiedName) { - - Map sortKeyOrder = mapOf(LEXICOGRAPHICAL_SORT_ORDER, mapOf("order", "desc")); - Map scoreSortOrder = mapOf("_score", mapOf("order", "desc")); - Map displayNameSortOrder = mapOf("displayName.keyword", mapOf("order", "desc")); + public static Map createDSLforCheckingPreExistingLexoRank(boolean isTerm, String lexoRank, String glossaryQualifiedName, String parentQualifiedName) { - Object[] sortArray = {sortKeyOrder, scoreSortOrder, displayNameSortOrder}; - - Map functionScore = mapOf("query", buildBoolQueryDuplicateLexoRank(lexoRank, glossaryQualifiedName, parentQualifiedName)); + Map boolMap = buildBoolQueryDuplicateLexoRank(isTerm, lexoRank, glossaryQualifiedName, parentQualifiedName); Map dsl = new HashMap<>(); dsl.put("from", 0); - dsl.put("size", 100); - dsl.put("sort", sortArray); - dsl.put("query", mapOf("function_score", functionScore)); + dsl.put("size", 1); + dsl.put("query", mapOf("bool", boolMap)); return dsl; } - private static Map buildBoolQueryDuplicateLexoRank(String lexoRank, String glossaryQualifiedName, String parentQualifiedName) { - Map boolQuery = new HashMap<>(); + private static Map buildBoolQueryDuplicateLexoRank(boolean isTerm, String lexoRank, String glossaryQualifiedName, String parentQualifiedName) { Map boolFilter = new HashMap<>(); List> mustArray = new ArrayList<>(); - List> mustNotArray = new ArrayList<>(); mustArray.add(mapOf("term", mapOf("__state", "ACTIVE"))); mustArray.add(mapOf("term", mapOf(LEXICOGRAPHICAL_SORT_ORDER, lexoRank))); if(StringUtils.isNotEmpty(glossaryQualifiedName)) { - mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossaryTerm", "AtlasGlossaryCategory")))); + mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList(ATLAS_GLOSSARY_TERM_TYPENAME, ATLAS_GLOSSARY_CATEGORY_TYPENAME)))); mustArray.add(mapOf("term", mapOf("__glossary", glossaryQualifiedName))); } else{ - mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossary")))); + mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList(ATLAS_GLOSSARY_ENTITY_TYPE)))); } if(StringUtils.isEmpty(parentQualifiedName)) { - mustNotArray.add(mapOf("exists", mapOf("field", "__categories"))); - mustNotArray.add(mapOf("exists", mapOf("field", "__parentCategory"))); + List> mustNotArray = new ArrayList<>(); + if(isTerm) { + mustNotArray.add(mapOf("exists", mapOf("field", "__categories"))); + } else { + mustNotArray.add(mapOf("exists", mapOf("field", "__parentCategory"))); + } boolFilter.put("must_not", mustNotArray); } else { List> shouldParentArray = new ArrayList<>(); - shouldParentArray.add(mapOf("term", mapOf("__categories", parentQualifiedName))); - shouldParentArray.add(mapOf("term", mapOf("__parentCategory", parentQualifiedName))); + if(isTerm) { + shouldParentArray.add(mapOf("term", mapOf("__categories", parentQualifiedName))); + } else { + shouldParentArray.add(mapOf("term", mapOf("__parentCategory", parentQualifiedName))); + } mustArray.add(mapOf("bool",mapOf("should", shouldParentArray))); } boolFilter.put("must", mustArray); - Map nestedBoolQuery = mapOf("bool", boolFilter); - - Map topBoolFilter = mapOf("filter", nestedBoolQuery); - - boolQuery.put("bool", topBoolFilter); - - return boolQuery; + return boolFilter; } - public static Map generateDSLQueryForLastCategory(String glossaryQualifiedName, String parentQualifiedName, boolean isTerm) { + public static Map generateDSLQueryForLastChild(String glossaryQualifiedName, String parentQualifiedName, boolean isTerm) { Map sortKeyOrder = mapOf(LEXICOGRAPHICAL_SORT_ORDER, mapOf("order", "desc")); - Map scoreSortOrder = mapOf("_score", mapOf("order", "desc")); - Map displayNameSortOrder = mapOf("displayName.keyword", mapOf("order", "desc")); - Object[] sortArray = {sortKeyOrder, scoreSortOrder, displayNameSortOrder}; + Object[] sortArray = {sortKeyOrder}; - Map functionScore = mapOf("query", buildBoolQuery(glossaryQualifiedName, parentQualifiedName, isTerm)); + Map boolMap = buildBoolQuery(glossaryQualifiedName, parentQualifiedName, isTerm); Map dsl = new HashMap<>(); dsl.put("from", 0); dsl.put("size", 1); dsl.put("sort", sortArray); - dsl.put("query", mapOf("function_score", functionScore)); + dsl.put("query", mapOf("bool", boolMap)); return dsl; } private static Map buildBoolQuery(String glossaryQualifiedName, String parentQualifiedName, boolean isTerm) { - Map boolQuery = new HashMap<>(); - int mustArrayLength = 0; - if(StringUtils.isEmpty(parentQualifiedName) && StringUtils.isEmpty(glossaryQualifiedName)){ - mustArrayLength = 2; - } else if(StringUtils.isEmpty(parentQualifiedName) && StringUtils.isNotEmpty(glossaryQualifiedName)){ - mustArrayLength = 3; - } else { - mustArrayLength = 4; - } - Map[] mustArray = new Map[mustArrayLength]; Map boolFilter = new HashMap<>(); - Map[] mustNotArray = new Map[2]; + List> mustArray = new ArrayList<>(); + List> mustNotArray = new ArrayList<>(); - mustArray[0] = mapOf("term", mapOf("__state", "ACTIVE")); + mustArray.add(mapOf("term", mapOf("__state", "ACTIVE"))); if(StringUtils.isNotEmpty(glossaryQualifiedName)) { String typeName = isTerm ? "AtlasGlossaryTerm" : "AtlasGlossaryCategory"; - mustArray[1] = mapOf("term", mapOf("__typeName.keyword", typeName)); - mustArray[2] = mapOf("term", mapOf("__glossary", glossaryQualifiedName)); + mustArray.add(mapOf("term", mapOf("__typeName.keyword", typeName))); + mustArray.add(mapOf("term", mapOf("__glossary", glossaryQualifiedName))); } else{ - mustArray[1] = mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossary"))); + mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossary")))); } if(StringUtils.isEmpty(parentQualifiedName)) { - mustNotArray[0] = mapOf("exists", mapOf("field", "__categories")); - mustNotArray[1] = mapOf("exists", mapOf("field", "__parentCategory")); + mustNotArray.add(mapOf("exists", mapOf("field", "__categories"))); + mustNotArray.add(mapOf("exists", mapOf("field", "__parentCategory"))); boolFilter.put("must_not", mustNotArray); } else { Map[] shouldParentArray = new Map[2]; shouldParentArray[0] = mapOf("term", mapOf("__categories", parentQualifiedName)); shouldParentArray[1] = mapOf("term", mapOf("__parentCategory", parentQualifiedName)); - mustArray[3] = mapOf("bool",mapOf("should", shouldParentArray)); + mustArray.add(mapOf("bool",mapOf("should", shouldParentArray))); } boolFilter.put("must", mustArray); - Map nestedBoolQuery = mapOf("bool", boolFilter); - - Map topBoolFilter = mapOf("filter", nestedBoolQuery); - - boolQuery.put("bool", topBoolFilter); - - return boolQuery; + return boolFilter; } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java index baac9c82c3..c955bdd5b4 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java @@ -134,7 +134,7 @@ private void processCreateCategory(AtlasEntity entity, AtlasVertex vertex) throw if(StringUtils.isEmpty(lexicographicalSortOrder)){ assignNewLexicographicalSortOrder(entity,glossaryQualifiedName, parentQname, this.discovery); } else { - isValidLexoRank(lexicographicalSortOrder, glossaryQualifiedName, parentQname, this.discovery); + isValidLexoRank(false, lexicographicalSortOrder, glossaryQualifiedName, parentQname, this.discovery); } entity.setAttribute(QUALIFIED_NAME, createQualifiedName(vertex)); @@ -168,7 +168,7 @@ private void processUpdateCategory(AtlasEntity entity, AtlasVertex vertex) throw parentQname = (String) parentCategory.getAttribute(QUALIFIED_NAME); } if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { - isValidLexoRank(lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery); + isValidLexoRank(false, lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery); } if (!currentGlossaryQualifiedName.equals(newGlossaryQualifiedName)){ diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java index b585705284..20d14f294a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java @@ -95,7 +95,7 @@ private void processCreateGlossary(AtlasStruct entity) throws AtlasBaseException if(StringUtils.isEmpty(lexicographicalSortOrder)) { assignNewLexicographicalSortOrder((AtlasEntity) entity, null, null, this.discovery); } else { - isValidLexoRank(lexicographicalSortOrder, "", "", this.discovery); + isValidLexoRank(false, lexicographicalSortOrder, "", "", this.discovery); } entity.setAttribute(QUALIFIED_NAME, createQualifiedName()); @@ -116,7 +116,7 @@ private void processUpdateGlossary(AtlasStruct entity, AtlasVertex vertex) throw } String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { - isValidLexoRank(lexicographicalSortOrder, "", "", this.discovery); + isValidLexoRank(false, lexicographicalSortOrder, "", "", this.discovery); } String vertexQnName = vertex.getProperty(QUALIFIED_NAME, String.class); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java index 9af03cabf8..32d258b224 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java @@ -104,7 +104,7 @@ private void processCreateTerm(AtlasEntity entity, AtlasVertex vertex) throws At if(StringUtils.isEmpty(lexicographicalSortOrder)){ assignNewLexicographicalSortOrder(entity, glossaryQName, parentQname, this.discovery); } else { - isValidLexoRank(lexicographicalSortOrder, glossaryQName, parentQname, this.discovery); + isValidLexoRank(true, lexicographicalSortOrder, glossaryQName, parentQname, this.discovery); } entity.setAttribute(QUALIFIED_NAME, createQualifiedName()); @@ -137,7 +137,7 @@ private void processUpdateTerm(AtlasEntity entity, AtlasVertex vertex) throws At String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { - isValidLexoRank(lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery); + isValidLexoRank(true, lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery); } if (!currentGlossaryQualifiedName.equals(newGlossaryQualifiedName)){ diff --git a/repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem10.java b/repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem10.java deleted file mode 100644 index f2c84e1a50..0000000000 --- a/repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem10.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.apache.atlas.util.lexoRank.system; - -public class LexoNumeralSystem10 implements LexoNumeralSystem { - public String getName() { - return "Base10"; - } - - public int getBase() { - return 10; - } - - public char getPositiveChar() { - return '+'; - } - - public char getNegativeChar() { - return '-'; - } - - public char getRadixPointChar() { - return '.'; - } - - public int toDigit(char ch) { - if (ch >= '0' && ch <= '9') return ch - 48; - throw new IllegalArgumentException("Not valid digit: " + ch); - } - - public char toChar(int digit) { - return (char) (digit + 48); - } -} From c7d679134492bb44f44c900cbd36376a4aaac457 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Wed, 12 Jun 2024 22:56:34 +0530 Subject: [PATCH 12/13] Added caching for same ranks in same request, for prevention of duplicacy. --- .../graph/v2/preprocessor/PreProcessorUtils.java | 14 ++++++++++++-- .../glossary/CategoryPreProcessor.java | 4 ++-- .../glossary/GlossaryPreProcessor.java | 4 ++-- .../v2/preprocessor/glossary/TermPreProcessor.java | 4 ++-- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index 5e363035ed..389aeaea88 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -216,7 +216,7 @@ public static void verifyDuplicateAssetByName(String typeName, String assetName, } } - public static void isValidLexoRank(boolean isTerm, String input, String glossaryQualifiedName, String parentQualifiedName, EntityDiscoveryService discovery) throws AtlasBaseException { + public static void isValidLexoRank(String entityType, String input, String glossaryQualifiedName, String parentQualifiedName, EntityDiscoveryService discovery) throws AtlasBaseException { // TODO : To remove this after migration is successful on all tenants and custom-sort is successfully GA Pattern regex = Pattern.compile(LEXORANK_VALID_PATTERN); @@ -229,7 +229,15 @@ public static void isValidLexoRank(boolean isTerm, String input, String glossary if(requestFromMigration) { return; } - Map dslQuery = createDSLforCheckingPreExistingLexoRank(isTerm, input, glossaryQualifiedName, parentQualifiedName); + Map lexoRankCache = RequestContext.get().getLexoRankCache(); + if(Objects.isNull(lexoRankCache)) { + lexoRankCache = new HashMap<>(); + } + String cacheKey = entityType + "-" + glossaryQualifiedName + "-" + parentQualifiedName; + if(lexoRankCache.containsKey(cacheKey)){ + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Duplicate Lexorank found"); + } + Map dslQuery = createDSLforCheckingPreExistingLexoRank(entityType.equals(ATLAS_GLOSSARY_TERM_TYPENAME), input, glossaryQualifiedName, parentQualifiedName); List assetsWithDuplicateRank = new ArrayList<>(); try { IndexSearchParams searchParams = new IndexSearchParams(); @@ -245,6 +253,8 @@ public static void isValidLexoRank(boolean isTerm, String input, String glossary throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Duplicate Lexorank found"); } + lexoRankCache.put(cacheKey, input); + RequestContext.get().setLexoRankCache(lexoRankCache); // TODO : Add the rebalancing logic here int colonIndex = input.indexOf(":"); if (colonIndex != -1 && input.substring(colonIndex + 1).length() >= REBALANCING_TRIGGER) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java index c955bdd5b4..a321b02fec 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java @@ -134,7 +134,7 @@ private void processCreateCategory(AtlasEntity entity, AtlasVertex vertex) throw if(StringUtils.isEmpty(lexicographicalSortOrder)){ assignNewLexicographicalSortOrder(entity,glossaryQualifiedName, parentQname, this.discovery); } else { - isValidLexoRank(false, lexicographicalSortOrder, glossaryQualifiedName, parentQname, this.discovery); + isValidLexoRank(entity.getTypeName(), lexicographicalSortOrder, glossaryQualifiedName, parentQname, this.discovery); } entity.setAttribute(QUALIFIED_NAME, createQualifiedName(vertex)); @@ -168,7 +168,7 @@ private void processUpdateCategory(AtlasEntity entity, AtlasVertex vertex) throw parentQname = (String) parentCategory.getAttribute(QUALIFIED_NAME); } if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { - isValidLexoRank(false, lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery); + isValidLexoRank(entity.getTypeName(), lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery); } if (!currentGlossaryQualifiedName.equals(newGlossaryQualifiedName)){ diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java index 20d14f294a..c07322300b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java @@ -95,7 +95,7 @@ private void processCreateGlossary(AtlasStruct entity) throws AtlasBaseException if(StringUtils.isEmpty(lexicographicalSortOrder)) { assignNewLexicographicalSortOrder((AtlasEntity) entity, null, null, this.discovery); } else { - isValidLexoRank(false, lexicographicalSortOrder, "", "", this.discovery); + isValidLexoRank(entity.getTypeName(), lexicographicalSortOrder, "", "", this.discovery); } entity.setAttribute(QUALIFIED_NAME, createQualifiedName()); @@ -116,7 +116,7 @@ private void processUpdateGlossary(AtlasStruct entity, AtlasVertex vertex) throw } String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { - isValidLexoRank(false, lexicographicalSortOrder, "", "", this.discovery); + isValidLexoRank(entity.getTypeName(), lexicographicalSortOrder, "", "", this.discovery); } String vertexQnName = vertex.getProperty(QUALIFIED_NAME, String.class); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java index 32d258b224..56c08df29b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java @@ -104,7 +104,7 @@ private void processCreateTerm(AtlasEntity entity, AtlasVertex vertex) throws At if(StringUtils.isEmpty(lexicographicalSortOrder)){ assignNewLexicographicalSortOrder(entity, glossaryQName, parentQname, this.discovery); } else { - isValidLexoRank(true, lexicographicalSortOrder, glossaryQName, parentQname, this.discovery); + isValidLexoRank(entity.getTypeName(), lexicographicalSortOrder, glossaryQName, parentQname, this.discovery); } entity.setAttribute(QUALIFIED_NAME, createQualifiedName()); @@ -137,7 +137,7 @@ private void processUpdateTerm(AtlasEntity entity, AtlasVertex vertex) throws At String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { - isValidLexoRank(true, lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery); + isValidLexoRank(entity.getTypeName(), lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery); } if (!currentGlossaryQualifiedName.equals(newGlossaryQualifiedName)){ From de75af0ee99c3b9b65e7ba98a1b556fb506bcfa1 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Fri, 14 Jun 2024 11:32:38 +0530 Subject: [PATCH 13/13] Fixed a minor bug where an empty lexo attribute in update call was removing the attribute from the vertex. --- .../store/graph/v2/preprocessor/PreProcessorUtils.java | 4 ++-- .../graph/v2/preprocessor/glossary/CategoryPreProcessor.java | 3 +++ .../graph/v2/preprocessor/glossary/GlossaryPreProcessor.java | 5 ++++- .../graph/v2/preprocessor/glossary/TermPreProcessor.java | 3 +++ 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index 389aeaea88..3e99be366e 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -222,8 +222,8 @@ public static void isValidLexoRank(String entityType, String input, String gloss Matcher matcher = regex.matcher(input); - if(!matcher.matches()){ - throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Invalid value for attribute"); + if(!matcher.matches() || StringUtils.isEmpty(input)){ + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Invalid value for lexicographicalSortOrder attribute"); } Boolean requestFromMigration = RequestContext.get().getRequestContextHeaders().getOrDefault("x-atlan-request-id", "").contains("custom-sort-migration"); if(requestFromMigration) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java index a321b02fec..74716d5f34 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java @@ -169,6 +169,9 @@ private void processUpdateCategory(AtlasEntity entity, AtlasVertex vertex) throw } if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { isValidLexoRank(entity.getTypeName(), lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery); + } else { + lexicographicalSortOrder = (String) storedCategory.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); + entity.setAttribute(LEXICOGRAPHICAL_SORT_ORDER, lexicographicalSortOrder); } if (!currentGlossaryQualifiedName.equals(newGlossaryQualifiedName)){ diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java index c07322300b..efdba02101 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java @@ -106,7 +106,7 @@ private void processUpdateGlossary(AtlasStruct entity, AtlasVertex vertex) throw AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdateGlossary"); String glossaryName = (String) entity.getAttribute(NAME); String vertexName = vertex.getProperty(NAME, String.class); - + AtlasEntity storedGlossary = entityRetriever.toAtlasEntity(vertex); if (!vertexName.equals(glossaryName) && glossaryExists(glossaryName)) { throw new AtlasBaseException(AtlasErrorCode.GLOSSARY_ALREADY_EXISTS,glossaryName); } @@ -117,6 +117,9 @@ private void processUpdateGlossary(AtlasStruct entity, AtlasVertex vertex) throw String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { isValidLexoRank(entity.getTypeName(), lexicographicalSortOrder, "", "", this.discovery); + } else { + lexicographicalSortOrder = (String) storedGlossary.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); + entity.setAttribute(LEXICOGRAPHICAL_SORT_ORDER, lexicographicalSortOrder); } String vertexQnName = vertex.getProperty(QUALIFIED_NAME, String.class); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java index 56c08df29b..253414e2db 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java @@ -138,6 +138,9 @@ private void processUpdateTerm(AtlasEntity entity, AtlasVertex vertex) throws At String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { isValidLexoRank(entity.getTypeName(), lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery); + } else { + lexicographicalSortOrder = (String) storedTerm.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); + entity.setAttribute(LEXICOGRAPHICAL_SORT_ORDER, lexicographicalSortOrder); } if (!currentGlossaryQualifiedName.equals(newGlossaryQualifiedName)){