From 49a6e6d6d6868dcbd9482f20d41be8f43996d1ad Mon Sep 17 00:00:00 2001 From: hr2904 Date: Wed, 22 May 2024 10:55:08 +0530 Subject: [PATCH 01/20] Init commmit, added lexorank generation util lib and logic for appending the lexo attribute to category. --- .../java/org/apache/atlas/type/Constants.java | 2 + .../AbstractGlossaryPreProcessor.java | 115 ++++++- .../glossary/CategoryPreProcessor.java | 15 +- .../glossary/GlossaryPreProcessor.java | 7 +- .../atlas/util/lexoRank/LexoDecimal.java | 178 ++++++++++ .../atlas/util/lexoRank/LexoInteger.java | 325 ++++++++++++++++++ .../apache/atlas/util/lexoRank/LexoRank.java | 287 ++++++++++++++++ .../atlas/util/lexoRank/LexoRankBucket.java | 83 +++++ .../lexoRank/system/LexoNumeralSystem.java | 18 + .../lexoRank/system/LexoNumeralSystem10.java | 32 ++ .../lexoRank/system/LexoNumeralSystem36.java | 36 ++ .../lexoRank/system/LexoNumeralSystem64.java | 41 +++ 12 files changed, 1125 insertions(+), 14 deletions(-) create mode 100644 repository/src/main/java/org/apache/atlas/util/lexoRank/LexoDecimal.java create mode 100644 repository/src/main/java/org/apache/atlas/util/lexoRank/LexoInteger.java create mode 100644 repository/src/main/java/org/apache/atlas/util/lexoRank/LexoRank.java create mode 100644 repository/src/main/java/org/apache/atlas/util/lexoRank/LexoRankBucket.java create mode 100644 repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem.java create mode 100644 repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem10.java create mode 100644 repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem36.java create mode 100644 repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem64.java diff --git a/intg/src/main/java/org/apache/atlas/type/Constants.java b/intg/src/main/java/org/apache/atlas/type/Constants.java index 01550ae9c0..effc5208b1 100644 --- a/intg/src/main/java/org/apache/atlas/type/Constants.java +++ b/intg/src/main/java/org/apache/atlas/type/Constants.java @@ -54,10 +54,12 @@ public final class Constants { public static final String GLOSSARY_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "glossary"); public static final String CATEGORIES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "categories"); public static final String CATEGORIES_PARENT_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "parentCategory"); + public static final String MEANINGS_TEXT_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "meaningsText"); public static final String MEANING_NAMES_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "meaningNames"); public static final String HAS_LINEAGE = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "hasLineage"); public static final String HAS_LINEAGE_VALID = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "hasLineageValid"); + public static final String LEXICOGRAPHICAL_SORT_ORDER = "lexicographicalSortOrder"; //Classification-Only System Attributes public static final String CLASSIFICATION_ENTITY_STATUS_PROPERTY_KEY = encodePropertyKey(INTERNAL_PROPERTY_KEY_PREFIX + "entityStatus"); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java index 08c604489c..1d263ed8d8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java @@ -26,7 +26,6 @@ import org.apache.atlas.authorize.AtlasPrivilege; import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; -import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasObjectId; @@ -41,17 +40,14 @@ import org.apache.atlas.tasks.TaskManagement; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.util.lexoRank.LexoRank; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; +import java.util.*; import java.util.stream.Collectors; import static org.apache.atlas.repository.Constants.ATLAS_GLOSSARY_TERM_ENTITY_TYPE; @@ -60,10 +56,8 @@ import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.indexSearchPaginated; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; -import static org.apache.atlas.type.Constants.MEANINGS_PROPERTY_KEY; -import static org.apache.atlas.type.Constants.MEANINGS_TEXT_PROPERTY_KEY; -import static org.apache.atlas.type.Constants.MEANING_NAMES_PROPERTY_KEY; -import static org.apache.atlas.type.Constants.PENDING_TASKS_PROPERTY_KEY; +import static org.apache.atlas.type.Constants.*; +import static org.apache.atlas.type.Constants.LEXICOGRAPHICAL_SORT_ORDER; public abstract class AbstractGlossaryPreProcessor implements PreProcessor { private static final Logger LOG = LoggerFactory.getLogger(AbstractGlossaryPreProcessor.class); @@ -73,6 +67,8 @@ public abstract class AbstractGlossaryPreProcessor implements PreProcessor { protected static final String ATTR_MEANINGS = "meanings"; protected static final String ATTR_CATEGORIES = "categories"; + protected static final String INIT_LEXORANK_OFFSET = "0|100000:"; + protected final AtlasTypeRegistry typeRegistry; protected final EntityGraphRetriever entityRetriever; protected final TaskManagement taskManagement; @@ -251,4 +247,103 @@ protected void recordUpdatedChildEntities(AtlasVertex entityVertex, Map attributes = new HashSet<>(); + attributes.add(LEXICOGRAPHICAL_SORT_ORDER); + List categories = null; + Map dslQuery = generateDSLQueryForLastCategory(glossaryQualifiedName, parentQualifiedName); + String lexoRank = ""; + try { + categories = indexSearchPaginated(dslQuery, attributes, this.discovery); + } catch (AtlasBaseException e) { + throw new RuntimeException(e); + } + + if (CollectionUtils.isNotEmpty(categories)) { + for (AtlasEntityHeader category : categories) { + String lexicographicalSortOrder = (String) category.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); + if(StringUtils.isNotEmpty(lexicographicalSortOrder)){ + LexoRank parsedLexoRank = LexoRank.parse(lexicographicalSortOrder); + LexoRank nextLexoRank = parsedLexoRank.genNext().genNext(); + lexoRank = nextLexoRank.toString(); + } else { + LexoRank parsedLexoRank = LexoRank.parse(INIT_LEXORANK_OFFSET); + LexoRank nextLexoRank = parsedLexoRank.genNext().genNext(); + lexoRank = nextLexoRank.toString(); + } + } + } else { + LexoRank parsedLexoRank = LexoRank.parse(INIT_LEXORANK_OFFSET); + LexoRank nextLexoRank = parsedLexoRank.genNext().genNext(); + lexoRank = nextLexoRank.toString(); + } + entity.setAttribute(LEXICOGRAPHICAL_SORT_ORDER, lexoRank); + } + + + public static Map generateDSLQueryForLastCategory(String glossaryQualifiedName, String parentQualifiedName) { + + Map sortKeyOrder = mapOf(LEXICOGRAPHICAL_SORT_ORDER, mapOf("order", "desc")); + Map scoreSortOrder = mapOf("_score", mapOf("order", "desc")); + Map displayNameSortOrder = mapOf("displayName.keyword", mapOf("order", "desc")); + + Object[] sortArray = {sortKeyOrder, scoreSortOrder, displayNameSortOrder}; + + Map functionScore = mapOf("query", buildBoolQuery(glossaryQualifiedName, parentQualifiedName)); + + Map dsl = new HashMap<>(); + dsl.put("from", 0); + dsl.put("size", 1); + dsl.put("sort", sortArray); + dsl.put("query", mapOf("function_score", functionScore)); + + return dsl; + } + + private static Map buildBoolQuery(String glossaryQualifiedName, String parentQualifiedName) { + Map boolQuery = new HashMap<>(); + int mustArrayLength = 0; + if(StringUtils.isEmpty(parentQualifiedName) && StringUtils.isEmpty(glossaryQualifiedName)){ + mustArrayLength = 2; + } else if(StringUtils.isEmpty(parentQualifiedName) && StringUtils.isNotEmpty(glossaryQualifiedName)){ + mustArrayLength = 3; + } else { + mustArrayLength = 4; + } + Map[] mustArray = new Map[mustArrayLength]; + Map boolFilter = new HashMap<>(); + Map[] mustNotArray = new Map[2]; + + mustArray[0] = mapOf("term", mapOf("__state", "ACTIVE")); + if(StringUtils.isNotEmpty(glossaryQualifiedName)) { + mustArray[1] = mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossaryTerm", "AtlasGlossaryCategory"))); + mustArray[2] = mapOf("term", mapOf("__glossary", glossaryQualifiedName)); + } else{ + mustArray[1] = mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossary"))); + } + + if(StringUtils.isEmpty(parentQualifiedName)) { + mustNotArray[0] = mapOf("exists", mapOf("field", "__categories")); + mustNotArray[1] = mapOf("exists", mapOf("field", "__parentCategory")); + boolFilter.put("must_not", mustNotArray); + } + else { + Map[] shouldParentArray = new Map[2]; + shouldParentArray[0] = mapOf("term", mapOf("__categories", parentQualifiedName)); + shouldParentArray[1] = mapOf("term", mapOf("__parentCategory", parentQualifiedName)); + mustArray[3] = mapOf("bool",mapOf("should", shouldParentArray)); + } + + boolFilter.put("must", mustArray); + + Map nestedBoolQuery = mapOf("bool", boolFilter); + + Map topBoolFilter = mapOf("filter", nestedBoolQuery); + + boolQuery.put("bool", topBoolFilter); + + return boolQuery; + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java index 88f72d2f16..532538aa1b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java @@ -40,6 +40,7 @@ import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasStructType; import org.apache.atlas.type.AtlasTypeRegistry; +import org.apache.atlas.util.lexoRank.LexoRank; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; @@ -47,6 +48,7 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import java.util.*; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; @@ -70,9 +72,7 @@ import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.*; import static org.apache.atlas.repository.store.graph.v2.tasks.MeaningsTaskFactory.UPDATE_ENTITY_MEANINGS_ON_TERM_UPDATE; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; -import static org.apache.atlas.type.Constants.CATEGORIES_PARENT_PROPERTY_KEY; -import static org.apache.atlas.type.Constants.CATEGORIES_PROPERTY_KEY; -import static org.apache.atlas.type.Constants.GLOSSARY_PROPERTY_KEY; +import static org.apache.atlas.type.Constants.*; public class CategoryPreProcessor extends AbstractGlossaryPreProcessor { private static final Logger LOG = LoggerFactory.getLogger(CategoryPreProcessor.class); @@ -117,12 +117,20 @@ public void processAttributes(AtlasStruct entityStruct, EntityMutationContext co private void processCreateCategory(AtlasEntity entity, AtlasVertex vertex) throws AtlasBaseException { AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processCreateCategory"); String catName = (String) entity.getAttribute(NAME); + String parentQname = null; if (StringUtils.isEmpty(catName) || isNameInvalid(catName)) { throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); } String glossaryQualifiedName = (String) anchor.getAttribute(QUALIFIED_NAME); + if (parentCategory != null) { + parentQname = (String) parentCategory.getAttribute(QUALIFIED_NAME); + } + String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); + if(StringUtils.isEmpty(lexicographicalSortOrder)){ + assignNewLexicographicalSortOrder(entity,glossaryQualifiedName, parentQname); + } categoryExists(catName, glossaryQualifiedName); validateParent(glossaryQualifiedName); @@ -489,4 +497,5 @@ private String createQualifiedName(AtlasVertex vertex) { return getUUID() + "@" + anchor.getAttribute(QUALIFIED_NAME); } + } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java index 6e3c962426..5cea0d0683 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java @@ -39,6 +39,7 @@ import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.getUUID; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.isNameInvalid; +import static org.apache.atlas.type.Constants.LEXICOGRAPHICAL_SORT_ORDER; public class GlossaryPreProcessor implements PreProcessor { private static final Logger LOG = LoggerFactory.getLogger(GlossaryPreProcessor.class); @@ -77,7 +78,11 @@ private void processCreateGlossary(AtlasStruct entity) throws AtlasBaseException if (StringUtils.isEmpty(glossaryName) || isNameInvalid(glossaryName)) { throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); } - + String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); + // TODO : Figure the placement for this method for glossary creation. +// if(StringUtils.isEmpty(lexicographicalSortOrder)){ +// assignNewLexicographicalSortOrder(entity,null, null); +// } if (glossaryExists(glossaryName)) { throw new AtlasBaseException(AtlasErrorCode.GLOSSARY_ALREADY_EXISTS,glossaryName); } diff --git a/repository/src/main/java/org/apache/atlas/util/lexoRank/LexoDecimal.java b/repository/src/main/java/org/apache/atlas/util/lexoRank/LexoDecimal.java new file mode 100644 index 0000000000..195970660a --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/util/lexoRank/LexoDecimal.java @@ -0,0 +1,178 @@ +package org.apache.atlas.util.lexoRank; + +import org.apache.atlas.util.lexoRank.system.LexoNumeralSystem; + +import java.util.Objects; + +public class LexoDecimal implements Comparable { + + private final LexoInteger mag; + private final int sig; + + private LexoDecimal(LexoInteger mag, int sig) { + this.mag = mag; + this.sig = sig; + } + + public static LexoDecimal half(LexoNumeralSystem sys) { + int mid = sys.getBase() / 2; + return make(LexoInteger.make(sys, 1, new int[] {mid}), 1); + } + + public static LexoDecimal parse(String str, LexoNumeralSystem system) { + int partialIndex = str.indexOf(system.getRadixPointChar()); + if (str.lastIndexOf(system.getRadixPointChar()) != partialIndex) + throw new IllegalArgumentException("More than one " + system.getRadixPointChar()); + + if (partialIndex < 0) return make(LexoInteger.parse(str, system), 0); + + String intStr = str.substring(0, partialIndex) + str.substring(partialIndex + 1); + return make(LexoInteger.parse(intStr, system), str.length() - 1 - partialIndex); + } + + public static LexoDecimal from(LexoInteger integer) { + return make(integer, 0); + } + + public static LexoDecimal make(LexoInteger integer, int sig) { + if (integer.isZero()) return new LexoDecimal(integer, 0); + + int zeroCount = 0; + + for (int i = 0; i < sig && integer.getMag(i) == 0; ++i) ++zeroCount; + + LexoInteger newInteger = integer.shiftRight(zeroCount); + int newSig = sig - zeroCount; + return new LexoDecimal(newInteger, newSig); + } + + public LexoNumeralSystem getSystem() { + return mag.getSystem(); + } + + public LexoDecimal add(LexoDecimal other) { + LexoInteger tMag = mag; + int tSig = sig; + LexoInteger oMag = other.mag; + + int oSig; + for (oSig = other.sig; tSig < oSig; ++tSig) tMag = tMag.shiftLeft(); + + while (tSig > oSig) { + oMag = oMag.shiftLeft(); + ++oSig; + } + + return make(tMag.add(oMag), tSig); + } + + public LexoDecimal subtract(LexoDecimal other) { + LexoInteger thisMag = mag; + int thisSig = sig; + LexoInteger otherMag = other.mag; + + int otherSig; + for (otherSig = other.sig; thisSig < otherSig; ++thisSig) thisMag = thisMag.shiftLeft(); + + while (thisSig > otherSig) { + otherMag = otherMag.shiftLeft(); + ++otherSig; + } + + return make(thisMag.subtract(otherMag), thisSig); + } + + public LexoDecimal multiply(LexoDecimal other) { + return make(mag.multiply(other.mag), sig + other.sig); + } + + public LexoInteger floor() { + return mag.shiftRight(sig); + } + + public LexoInteger ceil() { + if (isExact()) return mag; + + LexoInteger floor = floor(); + return floor.add(LexoInteger.one(floor.getSystem())); + } + + public boolean isExact() { + if (sig == 0) return true; + + for (int i = 0; i < sig; ++i) if (mag.getMag(i) != 0) return false; + + return true; + } + + public int getScale() { + return sig; + } + + public LexoDecimal setScale(int nSig) { + return setScale(nSig, false); + } + + public LexoDecimal setScale(int nSig, boolean ceiling) { + if (nSig >= sig) return this; + + if (nSig < 0) nSig = 0; + + int diff = sig - nSig; + LexoInteger nmag = mag.shiftRight(diff); + if (ceiling) nmag = nmag.add(LexoInteger.one(nmag.getSystem())); + + return make(nmag, nSig); + } + + public String format() { + String intStr = mag.format(); + if (sig == 0) return intStr; + + StringBuilder sb = new StringBuilder(intStr); + char head = sb.charAt(0); + boolean specialHead = + head == mag.getSystem().getPositiveChar() || head == mag.getSystem().getNegativeChar(); + if (specialHead) sb.delete(0, 1); + + while (sb.length() < sig + 1) sb.insert(0, mag.getSystem().toChar(0)); + + sb.insert(sb.length() - sig, mag.getSystem().getRadixPointChar()); + if (sb.length() - sig == 0) sb.insert(0, mag.getSystem().toChar(0)); + + if (specialHead) sb.insert(0, head); + + return sb.toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + LexoDecimal that = (LexoDecimal) o; + return sig == that.sig && Objects.equals(mag, that.mag); + } + + @Override + public int hashCode() { + return Objects.hash(mag, sig); + } + + @Override + public String toString() { + return format(); + } + + @Override + public int compareTo(LexoDecimal lexoDecimal) { + if (Objects.equals(this, lexoDecimal)) return 0; + if (Objects.equals(null, lexoDecimal)) return 1; + + LexoInteger tMag = mag; + LexoInteger oMag = lexoDecimal.mag; + if (sig > lexoDecimal.sig) oMag = oMag.shiftLeft(sig - lexoDecimal.sig); + else if (sig < lexoDecimal.sig) tMag = tMag.shiftLeft(lexoDecimal.sig - sig); + + return tMag.compareTo(oMag); + } +} diff --git a/repository/src/main/java/org/apache/atlas/util/lexoRank/LexoInteger.java b/repository/src/main/java/org/apache/atlas/util/lexoRank/LexoInteger.java new file mode 100644 index 0000000000..187797a20b --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/util/lexoRank/LexoInteger.java @@ -0,0 +1,325 @@ +package org.apache.atlas.util.lexoRank; + + +import org.apache.atlas.util.lexoRank.system.LexoNumeralSystem; + +import java.util.Arrays; +import java.util.Objects; + +public class LexoInteger implements Comparable { + private static final int[] ZERO_MAG = {0}; + private static final int[] ONE_MAG = {1}; + private final int negativeSign = -1; + private final int zeroSign = 0; + private final int positiveSign = 1; + private final int[] mag; + private final int sign; + private final LexoNumeralSystem sys; + + private LexoInteger(LexoNumeralSystem system, int sign, int[] mag) { + sys = system; + this.sign = sign; + this.mag = mag; + } + + private static int[] add(LexoNumeralSystem sys, int[] l, int[] r) { + int estimatedSize = Math.max(l.length, r.length); + int[] result = new int[estimatedSize]; + int carry = 0; + + for (int i = 0; i < estimatedSize; ++i) { + int lNum = i < l.length ? l[i] : 0; + int rNum = i < r.length ? r[i] : 0; + int sum = lNum + rNum + carry; + + for (carry = 0; sum >= sys.getBase(); sum -= sys.getBase()) ++carry; + + result[i] = sum; + } + + return extendWithCarry(result, carry); + } + + private static int[] extendWithCarry(int[] mag, int carry) { + int[] result = mag; + if (carry > 0) { + int[] extendedMag = new int[mag.length + 1]; + System.arraycopy(mag, 0, extendedMag, 0, mag.length); + extendedMag[extendedMag.length - 1] = carry; + result = extendedMag; + } + + return result; + } + + private static int[] subtract(LexoNumeralSystem sys, int[] l, int[] r) { + int[] rComplement = complement(sys, r, l.length); + int[] rSum = add(sys, l, rComplement); + rSum[rSum.length - 1] = 0; + return add(sys, rSum, ONE_MAG); + } + + private static int[] multiply(LexoNumeralSystem sys, int[] l, int[] r) { + int[] result = new int[l.length + r.length]; + + for (int li = 0; li < l.length; ++li) + for (int ri = 0; ri < r.length; ++ri) { + int resultIndex = li + ri; + + for (result[resultIndex] += l[li] * r[ri]; + result[resultIndex] >= sys.getBase(); + result[resultIndex] -= sys.getBase()) ++result[resultIndex + 1]; + } + + return result; + } + + private static int[] complement(LexoNumeralSystem sys, int[] mag, int digits) { + if (digits <= 0) throw new IllegalArgumentException("Expected at least 1 digit"); + + int[] nmag = new int[digits]; + + Arrays.fill(nmag, sys.getBase() - 1); + + for (int i = 0; i < mag.length; ++i) nmag[i] = sys.getBase() - 1 - mag[i]; + + return nmag; + } + + private static int compare(int[] l, int[] r) { + if (l.length < r.length) return -1; + + if (l.length > r.length) return 1; + + for (int i = l.length - 1; i >= 0; --i) { + if (l[i] < r[i]) return -1; + + if (l[i] > r[i]) return 1; + } + + return 0; + } + + public static LexoInteger parse(String strFull, LexoNumeralSystem system) { + String str = strFull; + int sign = 1; + if (strFull.indexOf(system.getPositiveChar()) == 0) { + str = strFull.substring(1); + } else if (strFull.indexOf(system.getNegativeChar()) == 0) { + str = strFull.substring(1); + sign = -1; + } + + int[] mag = new int[str.length()]; + int strIndex = mag.length - 1; + + for (int magIndex = 0; strIndex >= 0; ++magIndex) { + mag[magIndex] = system.toDigit(str.charAt(strIndex)); + --strIndex; + } + + return make(system, sign, mag); + } + + protected static LexoInteger zero(LexoNumeralSystem sys) { + return new LexoInteger(sys, 0, ZERO_MAG); + } + + protected static LexoInteger one(LexoNumeralSystem sys) { + return make(sys, 1, ONE_MAG); + } + + public static LexoInteger make(LexoNumeralSystem sys, int sign, int[] mag) { + int actualLength; + actualLength = mag.length; + while (actualLength > 0 && mag[actualLength - 1] == 0) { + --actualLength; + } + + if (actualLength == 0) return zero(sys); + + if (actualLength == mag.length) return new LexoInteger(sys, sign, mag); + + int[] nmag = new int[actualLength]; + System.arraycopy(mag, 0, nmag, 0, actualLength); + return new LexoInteger(sys, sign, nmag); + } + + public LexoInteger add(LexoInteger other) { + checkSystem(other); + if (isZero()) return other; + + if (other.isZero()) return this; + + if (sign != other.sign) { + LexoInteger pos; + if (sign == -1) { + pos = negate(); + LexoInteger val = pos.subtract(other); + return val.negate(); + } + + pos = other.negate(); + return subtract(pos); + } + + int[] result = add(sys, mag, other.mag); + return make(sys, sign, result); + } + + public LexoInteger subtract(LexoInteger other) { + checkSystem(other); + if (isZero()) return other.negate(); + + if (other.isZero()) return this; + + if (sign != other.sign) { + LexoInteger negate; + if (sign == -1) { + negate = negate(); + LexoInteger sum = negate.add(other); + return sum.negate(); + } + + negate = other.negate(); + return add(negate); + } + + int cmp = compare(mag, other.mag); + if (cmp == 0) return zero(sys); + + return cmp < 0 + ? make(sys, sign == -1 ? 1 : -1, subtract(sys, other.mag, mag)) + : make(sys, sign == -1 ? -1 : 1, subtract(sys, mag, other.mag)); + } + + public LexoInteger multiply(LexoInteger other) { + checkSystem(other); + if (isZero()) return this; + + if (other.isZero()) return other; + + if (isOneish()) return sign == other.sign ? make(sys, 1, other.mag) : make(sys, -1, other.mag); + + if (other.isOneish()) return sign == other.sign ? make(sys, 1, mag) : make(sys, -1, mag); + + int[] newMag = multiply(sys, mag, other.mag); + return sign == other.sign ? make(sys, 1, newMag) : make(sys, -1, newMag); + } + + public LexoInteger negate() { + return isZero() ? this : make(sys, sign == 1 ? -1 : 1, mag); + } + + public LexoInteger shiftLeft() { + return shiftLeft(1); + } + + public LexoInteger shiftLeft(int times) { + if (times == 0) return this; + + if (times < 0) return shiftRight(Math.abs(times)); + + int[] nmag = new int[mag.length + times]; + System.arraycopy(mag, 0, nmag, times, mag.length); + return make(sys, sign, nmag); + } + + public LexoInteger shiftRight() { + return shiftRight(1); + } + + public LexoInteger shiftRight(int times) { + if (mag.length - times <= 0) return zero(sys); + + int[] nmag = new int[mag.length - times]; + System.arraycopy(mag, times, nmag, 0, nmag.length); + return make(sys, sign, nmag); + } + + public LexoInteger complement() { + return complement(mag.length); + } + + private LexoInteger complement(int digits) { + return make(sys, sign, complement(sys, mag, digits)); + } + + public boolean isZero() { + return sign == 0 && mag.length == 1 && mag[0] == 0; + } + + private boolean isOneish() { + return mag.length == 1 && mag[0] == 1; + } + + public boolean isOne() { + return sign == 1 && mag.length == 1 && mag[0] == 1; + } + + public int getMag(int index) { + return mag[index]; + } + + public LexoNumeralSystem getSystem() { + return sys; + } + + private void checkSystem(LexoInteger other) { + if (!sys.getName().equals(other.sys.getName())) + throw new IllegalArgumentException("Expected numbers of same numeral sys"); + } + + public String format() { + if (isZero()) return String.valueOf(sys.toChar(0)); + StringBuilder sb = new StringBuilder(); + for (int digit : mag) { + sb.insert(0, sys.toChar(digit)); + } + if (sign == -1) sb.setCharAt(0, sys.getNegativeChar()); + + return sb.toString(); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + LexoInteger that = (LexoInteger) o; + return sign == that.sign && Arrays.equals(mag, that.mag) && Objects.equals(sys, that.sys); + } + + @Override + public int hashCode() { + int result = Objects.hash(negativeSign, zeroSign, positiveSign, sign, sys); + result = 31 * result + Arrays.hashCode(mag); + return result; + } + + @Override + public String toString() { + return format(); + } + + @Override + public int compareTo(LexoInteger lexoInteger) { + if (this.equals(lexoInteger)) return 0; + if (null == lexoInteger) return 1; + + if (sign == -1) { + if (lexoInteger.sign == -1) { + int cmp = compare(mag, lexoInteger.mag); + if (cmp == -1) return 1; + return cmp == 1 ? -1 : 0; + } + + return -1; + } + + if (sign == 1) return lexoInteger.sign == 1 ? compare(mag, lexoInteger.mag) : 1; + + if (lexoInteger.sign == -1) return 1; + + return lexoInteger.sign == 1 ? -1 : 0; + } +} diff --git a/repository/src/main/java/org/apache/atlas/util/lexoRank/LexoRank.java b/repository/src/main/java/org/apache/atlas/util/lexoRank/LexoRank.java new file mode 100644 index 0000000000..29a8bf2d41 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/util/lexoRank/LexoRank.java @@ -0,0 +1,287 @@ +package org.apache.atlas.util.lexoRank; + + + + +import org.apache.atlas.util.lexoRank.system.LexoNumeralSystem; +import org.apache.atlas.util.lexoRank.system.LexoNumeralSystem36; + +import java.util.Objects; + +public class LexoRank implements Comparable { + + public static final LexoNumeralSystem NUMERAL_SYSTEM = new LexoNumeralSystem36(); + private static final LexoDecimal ZERO_DECIMAL = LexoDecimal.parse("0", NUMERAL_SYSTEM); + private static final LexoDecimal ONE_DECIMAL = LexoDecimal.parse("1", NUMERAL_SYSTEM); + private static final LexoDecimal EIGHT_DECIMAL = LexoDecimal.parse("8", NUMERAL_SYSTEM); + private static final LexoDecimal MIN_DECIMAL = ZERO_DECIMAL; + + private static final LexoDecimal MAX_DECIMAL = + LexoDecimal.parse("1000000", NUMERAL_SYSTEM).subtract(ONE_DECIMAL); + + private static final LexoDecimal MID_DECIMAL = between(MIN_DECIMAL, MAX_DECIMAL); + private static final LexoDecimal INITIAL_MIN_DECIMAL = + LexoDecimal.parse("100000", NUMERAL_SYSTEM); + + private static final LexoDecimal INITIAL_MAX_DECIMAL = + LexoDecimal.parse( + NUMERAL_SYSTEM.toChar(NUMERAL_SYSTEM.getBase() - 2) + "00000", NUMERAL_SYSTEM); + + private final String value; + private final LexoRankBucket bucket; + private final LexoDecimal decimal; + + private LexoRank(String value) { + this.value = value; + String[] parts = this.value.split("\\|"); + bucket = LexoRankBucket.from(parts[0]); + decimal = LexoDecimal.parse(parts[1], NUMERAL_SYSTEM); + } + + private LexoRank(LexoRankBucket bucket, LexoDecimal dec) { + value = bucket.format() + "|" + formatDecimal(dec); + this.bucket = bucket; + decimal = dec; + } + + public static LexoRank min() { + return from(LexoRankBucket.BUCKET_0, MIN_DECIMAL); + } + + public static LexoRank max() { + return max(LexoRankBucket.BUCKET_0); + } + + public static LexoRank middle() { + LexoRank minLexoRank = min(); + return minLexoRank.between(max(minLexoRank.bucket)); + } + + public static LexoRank max(LexoRankBucket bucket) { + return from(bucket, MAX_DECIMAL); + } + + public static LexoRank initial(LexoRankBucket bucket) { + return bucket == LexoRankBucket.BUCKET_0 + ? from(bucket, INITIAL_MIN_DECIMAL) + : from(bucket, INITIAL_MAX_DECIMAL); + } + + private static LexoDecimal between(LexoDecimal oLeft, LexoDecimal oRight) { + if (oLeft.getSystem() != oRight.getSystem()) + throw new IllegalArgumentException("Expected same system"); + + LexoDecimal left = oLeft; + LexoDecimal right = oRight; + LexoDecimal nLeft; + if (oLeft.getScale() < oRight.getScale()) { + nLeft = oRight.setScale(oLeft.getScale(), false); + if (oLeft.compareTo(nLeft) >= 0) return middle(oLeft, oRight); + + right = nLeft; + } + + if (oLeft.getScale() > right.getScale()) { + nLeft = oLeft.setScale(right.getScale(), true); + if (nLeft.compareTo(right) >= 0) return middle(oLeft, oRight); + + left = nLeft; + } + + LexoDecimal nRight; + for (int scale = left.getScale(); scale > 0; right = nRight) { + int nScale1 = scale - 1; + LexoDecimal nLeft1 = left.setScale(nScale1, true); + nRight = right.setScale(nScale1, false); + int cmp = nLeft1.compareTo(nRight); + if (cmp == 0) return checkMid(oLeft, oRight, nLeft1); + + if (nLeft1.compareTo(nRight) > 0) break; + + scale = nScale1; + left = nLeft1; + } + + LexoDecimal mid = middle(oLeft, oRight, left, right); + + int nScale; + for (int mScale = mid.getScale(); mScale > 0; mScale = nScale) { + nScale = mScale - 1; + LexoDecimal nMid = mid.setScale(nScale); + if (oLeft.compareTo(nMid) >= 0 || nMid.compareTo(oRight) >= 0) break; + + mid = nMid; + } + + return mid; + } + + private static LexoDecimal middle( + LexoDecimal lBound, LexoDecimal rBound, LexoDecimal left, LexoDecimal right) { + LexoDecimal mid = middle(left, right); + return checkMid(lBound, rBound, mid); + } + + private static LexoDecimal checkMid(LexoDecimal lBound, LexoDecimal rBound, LexoDecimal mid) { + if (lBound.compareTo(mid) >= 0) return middle(lBound, rBound); + + return mid.compareTo(rBound) >= 0 ? middle(lBound, rBound) : mid; + } + + private static LexoDecimal middle(LexoDecimal left, LexoDecimal right) { + LexoDecimal sum = left.add(right); + LexoDecimal mid = sum.multiply(LexoDecimal.half(left.getSystem())); + int scale = Math.max(left.getScale(), right.getScale()); + if (mid.getScale() > scale) { + LexoDecimal roundDown = mid.setScale(scale, false); + if (roundDown.compareTo(left) > 0) return roundDown; + + LexoDecimal roundUp = mid.setScale(scale, true); + if (roundUp.compareTo(right) < 0) return roundUp; + } + + return mid; + } + + private static String formatDecimal(LexoDecimal dec) { + String formatVal = dec.format(); + StringBuilder val = new StringBuilder(formatVal); + int partialIndex = formatVal.indexOf(NUMERAL_SYSTEM.getRadixPointChar()); + char zero = NUMERAL_SYSTEM.toChar(0); + if (partialIndex < 0) { + partialIndex = formatVal.length(); + val.append(NUMERAL_SYSTEM.getRadixPointChar()); + } + + while (partialIndex < 6) { + val.insert(0, zero); + ++partialIndex; + } + + // TODO CHECK LOGIC + int valLength = val.length() - 1; + while (val.charAt(valLength) == zero) { + valLength = val.length() - 1; + } + + return val.toString(); + } + + public static LexoRank parse(String str) { + if (isNullOrWhiteSpace(str)) throw new IllegalArgumentException(str); + return new LexoRank(str); + } + + public static LexoRank from(LexoRankBucket bucket, LexoDecimal dec) { + if (!dec.getSystem().getName().equals(NUMERAL_SYSTEM.getName())) + throw new IllegalArgumentException("Expected different system"); + + return new LexoRank(bucket, dec); + } + + private static boolean isNullOrWhiteSpace(String string) { + return string == null || string.equals(" "); + } + + public LexoRankBucket getBucket() { + return bucket; + } + + public LexoDecimal getDecimal() { + return decimal; + } + + public int CompareTo(LexoRank other) { + if (Objects.equals(this, other)) return 0; + if (Objects.equals(null, other)) return 1; + return value.compareTo(other.value); + } + + public LexoRank genPrev() { + if (isMax()) return new LexoRank(bucket, INITIAL_MAX_DECIMAL); + + LexoInteger floorInteger = decimal.floor(); + LexoDecimal floorDecimal = LexoDecimal.from(floorInteger); + LexoDecimal nextDecimal = floorDecimal.subtract(EIGHT_DECIMAL); + if (nextDecimal.compareTo(MIN_DECIMAL) <= 0) nextDecimal = between(MIN_DECIMAL, decimal); + + return new LexoRank(bucket, nextDecimal); + } + + public LexoRank inNextBucket() { + return from(bucket.next(), decimal); + } + + public LexoRank inPrevBucket() { + return from(bucket.prev(), decimal); + } + + public boolean isMin() { + return decimal.equals(MIN_DECIMAL); + } + + public boolean isMax() { + return decimal.equals(MAX_DECIMAL); + } + + public String format() { + return value; + } + + public LexoRank genNext() { + if (isMin()) return new LexoRank(bucket, INITIAL_MIN_DECIMAL); + + LexoInteger ceilInteger = decimal.ceil(); + LexoDecimal ceilDecimal = LexoDecimal.from(ceilInteger); + LexoDecimal nextDecimal = ceilDecimal.add(EIGHT_DECIMAL); + if (nextDecimal.compareTo(MAX_DECIMAL) >= 0) nextDecimal = between(decimal, MAX_DECIMAL); + + return new LexoRank(bucket, nextDecimal); + } + + public LexoRank between(LexoRank other) { + if (!bucket.equals(other.bucket)) + throw new IllegalArgumentException("Between works only within the same bucket"); + + int cmp = decimal.compareTo(other.decimal); + if (cmp > 0) return new LexoRank(bucket, between(other.decimal, decimal)); + if (cmp == 0) + throw new IllegalArgumentException( + "Try to rank between issues with same rank this=" + + this + + " other=" + + other + + " this.decimal=" + + decimal + + " other.decimal=" + + other.decimal); + return new LexoRank(bucket, between(decimal, other.decimal)); + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + LexoRank lexoRank = (LexoRank) o; + return Objects.equals(value, lexoRank.value) + && Objects.equals(bucket, lexoRank.bucket) + && Objects.equals(decimal, lexoRank.decimal); + } + + @Override + public int hashCode() { + return Objects.hash(value, bucket, decimal); + } + + @Override + public String toString() { + return format(); + } + + @Override + public int compareTo(LexoRank lexoRank) { + if (Objects.equals(this, lexoRank)) return 0; + if (Objects.equals(null, lexoRank)) return 1; + return value.compareTo(lexoRank.value); + } +} diff --git a/repository/src/main/java/org/apache/atlas/util/lexoRank/LexoRankBucket.java b/repository/src/main/java/org/apache/atlas/util/lexoRank/LexoRankBucket.java new file mode 100644 index 0000000000..c3b60b4982 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/util/lexoRank/LexoRankBucket.java @@ -0,0 +1,83 @@ +package org.apache.atlas.util.lexoRank; + + +import java.util.Objects; + +public class LexoRankBucket { + + protected static final LexoRankBucket BUCKET_0 = new LexoRankBucket("0"); + protected static final LexoRankBucket BUCKET_1 = new LexoRankBucket("1"); + protected static final LexoRankBucket BUCKET_2 = new LexoRankBucket("2"); + + private static final LexoRankBucket[] VALUES = {BUCKET_0, BUCKET_1, BUCKET_2}; + + private final LexoInteger value; + + private LexoRankBucket(String val) { + value = LexoInteger.parse(val, LexoRank.NUMERAL_SYSTEM); + } + + public static LexoRankBucket resolve(int bucketId) { + for (LexoRankBucket bucket : VALUES) { + if (bucket.equals(from(String.valueOf(bucketId)))) return bucket; + } + + throw new IllegalArgumentException("No bucket found with id " + bucketId); + } + + public static LexoRankBucket from(String str) { + LexoInteger val = LexoInteger.parse(str, LexoRank.NUMERAL_SYSTEM); + + for (LexoRankBucket bucket : VALUES) { + if (bucket.value.equals(val)) return bucket; + } + + throw new IllegalArgumentException("Unknown bucket: " + str); + } + + public static LexoRankBucket min() { + return VALUES[0]; + } + + public static LexoRankBucket max() { + return VALUES[VALUES.length - 1]; + } + + public String format() { + return value.format(); + } + + public LexoRankBucket next() { + if (this == BUCKET_0) return BUCKET_1; + + if (this == BUCKET_1) return BUCKET_2; + + return this == BUCKET_2 ? BUCKET_0 : BUCKET_2; + } + + public LexoRankBucket prev() { + if (this == BUCKET_0) return BUCKET_2; + + if (this == BUCKET_1) return BUCKET_0; + + return this == BUCKET_2 ? BUCKET_1 : BUCKET_0; + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + LexoRankBucket that = (LexoRankBucket) o; + return Objects.equals(value, that.value); + } + + @Override + public String toString() { + return format(); + } + + @Override + public int hashCode() { + return Objects.hash(value); + } +} diff --git a/repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem.java b/repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem.java new file mode 100644 index 0000000000..a996079e3d --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem.java @@ -0,0 +1,18 @@ +package org.apache.atlas.util.lexoRank.system; + +public interface LexoNumeralSystem { + + String getName(); + + int getBase(); + + char getPositiveChar(); + + char getNegativeChar(); + + char getRadixPointChar(); + + int toDigit(char var1); + + char toChar(int var1); +} diff --git a/repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem10.java b/repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem10.java new file mode 100644 index 0000000000..f2c84e1a50 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem10.java @@ -0,0 +1,32 @@ +package org.apache.atlas.util.lexoRank.system; + +public class LexoNumeralSystem10 implements LexoNumeralSystem { + public String getName() { + return "Base10"; + } + + public int getBase() { + return 10; + } + + public char getPositiveChar() { + return '+'; + } + + public char getNegativeChar() { + return '-'; + } + + public char getRadixPointChar() { + return '.'; + } + + public int toDigit(char ch) { + if (ch >= '0' && ch <= '9') return ch - 48; + throw new IllegalArgumentException("Not valid digit: " + ch); + } + + public char toChar(int digit) { + return (char) (digit + 48); + } +} diff --git a/repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem36.java b/repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem36.java new file mode 100644 index 0000000000..731871842e --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem36.java @@ -0,0 +1,36 @@ +package org.apache.atlas.util.lexoRank.system; + + +public class LexoNumeralSystem36 implements LexoNumeralSystem { + private final char[] digits = "0123456789abcdefghijklmnopqrstuvwxyz".toCharArray(); + + public String getName() { + return "Base36"; + } + + public int getBase() { + return 36; + } + + public char getPositiveChar() { + return '+'; + } + + public char getNegativeChar() { + return '-'; + } + + public char getRadixPointChar() { + return ':'; + } + + public int toDigit(char ch) { + if (ch >= '0' && ch <= '9') return ch - 48; + if (ch >= 'a' && ch <= 'z') return ch - 97 + 10; + throw new IllegalArgumentException("Not valid digit: " + ch); + } + + public char toChar(int digit) { + return digits[digit]; + } +} diff --git a/repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem64.java b/repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem64.java new file mode 100644 index 0000000000..89b0af2390 --- /dev/null +++ b/repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem64.java @@ -0,0 +1,41 @@ +package org.apache.atlas.util.lexoRank.system; + + +public class LexoNumeralSystem64 implements LexoNumeralSystem { + + private final char[] digits = + "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ^_abcdefghijklmnopqrstuvwxyz".toCharArray(); + + public String getName() { + return "Base64"; + } + + public int getBase() { + return 64; + } + + public char getPositiveChar() { + return '+'; + } + + public char getNegativeChar() { + return '-'; + } + + public char getRadixPointChar() { + return ':'; + } + + public int toDigit(char ch) { + if (ch >= '0' && ch <= '9') return ch - 48; + if (ch >= 'A' && ch <= 'Z') return ch - 65 + 10; + if (ch == '^') return 36; + if (ch == '_') return 37; + if (ch >= 'a' && ch <= 'z') return ch - 97 + 38; + throw new IllegalArgumentException("Not valid digit: " + ch); + } + + public char toChar(int digit) { + return digits[digit]; + } +} From 7f36434f44646accee4ed1c5e00d4b39b4d19155 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Wed, 22 May 2024 18:19:36 +0530 Subject: [PATCH 02/20] Added method to append lexicographicSortOrder attribute to any Glossary based Entity that does not have it while creation. --- .../store/graph/v2/AtlasEntityStoreV2.java | 2 +- .../v2/preprocessor/PreProcessorUtils.java | 118 ++++++++++++++++++ .../AbstractGlossaryPreProcessor.java | 100 --------------- .../glossary/CategoryPreProcessor.java | 7 +- .../glossary/GlossaryPreProcessor.java | 23 ++-- .../glossary/TermPreProcessor.java | 17 ++- .../java/org/apache/atlas/RequestContext.java | 12 ++ 7 files changed, 164 insertions(+), 115 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java index 224f04970b..8ea3107f7b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/AtlasEntityStoreV2.java @@ -1859,7 +1859,7 @@ public List getPreProcessor(String typeName) { switch (typeName) { case ATLAS_GLOSSARY_ENTITY_TYPE: - preProcessors.add(new GlossaryPreProcessor(typeRegistry, entityRetriever)); + preProcessors.add(new GlossaryPreProcessor(typeRegistry, entityRetriever, graph)); break; case ATLAS_GLOSSARY_TERM_ENTITY_TYPE: diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index 260f228351..111e4b2356 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -1,6 +1,7 @@ package org.apache.atlas.repository.store.graph.v2.preprocessor; import org.apache.atlas.AtlasErrorCode; +import org.apache.atlas.RequestContext; import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.discovery.IndexSearchParams; @@ -14,6 +15,7 @@ import org.apache.atlas.type.AtlasStructType; import org.apache.atlas.type.AtlasTypeRegistry; import org.apache.atlas.util.NanoIdUtils; +import org.apache.atlas.util.lexoRank.LexoRank; import org.apache.atlas.utils.AtlasEntityUtil; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; @@ -26,6 +28,7 @@ import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; +import static org.apache.atlas.type.Constants.LEXICOGRAPHICAL_SORT_ORDER; public class PreProcessorUtils { private static final Logger LOG = LoggerFactory.getLogger(PreProcessorUtils.class); @@ -39,6 +42,7 @@ public class PreProcessorUtils { public static final String CATEGORY_CHILDREN = "childrenCategories"; public static final String GLOSSARY_TERM_REL_TYPE = "AtlasGlossaryTermAnchor"; public static final String GLOSSARY_CATEGORY_REL_TYPE = "AtlasGlossaryCategoryAnchor"; + public static final String INIT_LEXORANK_OFFSET = "0|100000:"; //DataMesh models constants public static final String PARENT_DOMAIN_REL_TYPE = "parentDomain"; @@ -202,4 +206,118 @@ public static void verifyDuplicateAssetByName(String typeName, String assetName, throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, errorMessage); } } + + public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String glossaryQualifiedName, String parentQualifiedName, EntityDiscoveryService discovery) { + Map lexoRankCache = RequestContext.get().getLexoRankCache(); + + if(Objects.isNull(lexoRankCache)) { + lexoRankCache = new HashMap<>(); + } + String lexoRank = ""; + String lastLexoRank = ""; + + if(lexoRankCache.containsKey(glossaryQualifiedName + "-" + parentQualifiedName)) { + lastLexoRank = lexoRankCache.get(glossaryQualifiedName + "-" + parentQualifiedName); + + } else { + Set attributes = new HashSet<>(); + attributes.add(LEXICOGRAPHICAL_SORT_ORDER); + List categories = null; + Map dslQuery = generateDSLQueryForLastCategory(glossaryQualifiedName, parentQualifiedName); + try { + IndexSearchParams searchParams = new IndexSearchParams(); + searchParams.setAttributes(attributes); + searchParams.setDsl(dslQuery); + categories = discovery.directIndexSearch(searchParams).getEntities(); + } catch (AtlasBaseException e) { + e.printStackTrace(); + } + + if (CollectionUtils.isNotEmpty(categories)) { + for (AtlasEntityHeader category : categories) { + String lexicographicalSortOrder = (String) category.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); + if (StringUtils.isNotEmpty(lexicographicalSortOrder)) { + lastLexoRank = lexicographicalSortOrder; + } else { + lastLexoRank = INIT_LEXORANK_OFFSET; + } + } + } else { + lastLexoRank = INIT_LEXORANK_OFFSET; + } + } + + LexoRank parsedLexoRank = LexoRank.parse(lastLexoRank); + LexoRank nextLexoRank = parsedLexoRank.genNext().genNext(); + lexoRank = nextLexoRank.toString(); + + entity.setAttribute(LEXICOGRAPHICAL_SORT_ORDER, lexoRank); + lexoRankCache.put(glossaryQualifiedName + "-" + parentQualifiedName, lexoRank); + RequestContext.get().setLexoRankCache(lexoRankCache); + } + + + public static Map generateDSLQueryForLastCategory(String glossaryQualifiedName, String parentQualifiedName) { + + Map sortKeyOrder = mapOf(LEXICOGRAPHICAL_SORT_ORDER, mapOf("order", "desc")); + Map scoreSortOrder = mapOf("_score", mapOf("order", "desc")); + Map displayNameSortOrder = mapOf("displayName.keyword", mapOf("order", "desc")); + + Object[] sortArray = {sortKeyOrder, scoreSortOrder, displayNameSortOrder}; + + Map functionScore = mapOf("query", buildBoolQuery(glossaryQualifiedName, parentQualifiedName)); + + Map dsl = new HashMap<>(); + dsl.put("from", 0); + dsl.put("size", 1); + dsl.put("sort", sortArray); + dsl.put("query", mapOf("function_score", functionScore)); + + return dsl; + } + + private static Map buildBoolQuery(String glossaryQualifiedName, String parentQualifiedName) { + Map boolQuery = new HashMap<>(); + int mustArrayLength = 0; + if(StringUtils.isEmpty(parentQualifiedName) && StringUtils.isEmpty(glossaryQualifiedName)){ + mustArrayLength = 2; + } else if(StringUtils.isEmpty(parentQualifiedName) && StringUtils.isNotEmpty(glossaryQualifiedName)){ + mustArrayLength = 3; + } else { + mustArrayLength = 4; + } + Map[] mustArray = new Map[mustArrayLength]; + Map boolFilter = new HashMap<>(); + Map[] mustNotArray = new Map[2]; + + mustArray[0] = mapOf("term", mapOf("__state", "ACTIVE")); + if(StringUtils.isNotEmpty(glossaryQualifiedName)) { + mustArray[1] = mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossaryTerm", "AtlasGlossaryCategory"))); + mustArray[2] = mapOf("term", mapOf("__glossary", glossaryQualifiedName)); + } else{ + mustArray[1] = mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossary"))); + } + + if(StringUtils.isEmpty(parentQualifiedName)) { + mustNotArray[0] = mapOf("exists", mapOf("field", "__categories")); + mustNotArray[1] = mapOf("exists", mapOf("field", "__parentCategory")); + boolFilter.put("must_not", mustNotArray); + } + else { + Map[] shouldParentArray = new Map[2]; + shouldParentArray[0] = mapOf("term", mapOf("__categories", parentQualifiedName)); + shouldParentArray[1] = mapOf("term", mapOf("__parentCategory", parentQualifiedName)); + mustArray[3] = mapOf("bool",mapOf("should", shouldParentArray)); + } + + boolFilter.put("must", mustArray); + + Map nestedBoolQuery = mapOf("bool", boolFilter); + + Map topBoolFilter = mapOf("filter", nestedBoolQuery); + + boolQuery.put("bool", topBoolFilter); + + return boolQuery; + } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java index 1d263ed8d8..383273d73a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java @@ -67,7 +67,6 @@ public abstract class AbstractGlossaryPreProcessor implements PreProcessor { protected static final String ATTR_MEANINGS = "meanings"; protected static final String ATTR_CATEGORIES = "categories"; - protected static final String INIT_LEXORANK_OFFSET = "0|100000:"; protected final AtlasTypeRegistry typeRegistry; protected final EntityGraphRetriever entityRetriever; @@ -247,103 +246,4 @@ protected void recordUpdatedChildEntities(AtlasVertex entityVertex, Map attributes = new HashSet<>(); - attributes.add(LEXICOGRAPHICAL_SORT_ORDER); - List categories = null; - Map dslQuery = generateDSLQueryForLastCategory(glossaryQualifiedName, parentQualifiedName); - String lexoRank = ""; - try { - categories = indexSearchPaginated(dslQuery, attributes, this.discovery); - } catch (AtlasBaseException e) { - throw new RuntimeException(e); - } - - if (CollectionUtils.isNotEmpty(categories)) { - for (AtlasEntityHeader category : categories) { - String lexicographicalSortOrder = (String) category.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); - if(StringUtils.isNotEmpty(lexicographicalSortOrder)){ - LexoRank parsedLexoRank = LexoRank.parse(lexicographicalSortOrder); - LexoRank nextLexoRank = parsedLexoRank.genNext().genNext(); - lexoRank = nextLexoRank.toString(); - } else { - LexoRank parsedLexoRank = LexoRank.parse(INIT_LEXORANK_OFFSET); - LexoRank nextLexoRank = parsedLexoRank.genNext().genNext(); - lexoRank = nextLexoRank.toString(); - } - } - } else { - LexoRank parsedLexoRank = LexoRank.parse(INIT_LEXORANK_OFFSET); - LexoRank nextLexoRank = parsedLexoRank.genNext().genNext(); - lexoRank = nextLexoRank.toString(); - } - entity.setAttribute(LEXICOGRAPHICAL_SORT_ORDER, lexoRank); - } - - - public static Map generateDSLQueryForLastCategory(String glossaryQualifiedName, String parentQualifiedName) { - - Map sortKeyOrder = mapOf(LEXICOGRAPHICAL_SORT_ORDER, mapOf("order", "desc")); - Map scoreSortOrder = mapOf("_score", mapOf("order", "desc")); - Map displayNameSortOrder = mapOf("displayName.keyword", mapOf("order", "desc")); - - Object[] sortArray = {sortKeyOrder, scoreSortOrder, displayNameSortOrder}; - - Map functionScore = mapOf("query", buildBoolQuery(glossaryQualifiedName, parentQualifiedName)); - - Map dsl = new HashMap<>(); - dsl.put("from", 0); - dsl.put("size", 1); - dsl.put("sort", sortArray); - dsl.put("query", mapOf("function_score", functionScore)); - - return dsl; - } - - private static Map buildBoolQuery(String glossaryQualifiedName, String parentQualifiedName) { - Map boolQuery = new HashMap<>(); - int mustArrayLength = 0; - if(StringUtils.isEmpty(parentQualifiedName) && StringUtils.isEmpty(glossaryQualifiedName)){ - mustArrayLength = 2; - } else if(StringUtils.isEmpty(parentQualifiedName) && StringUtils.isNotEmpty(glossaryQualifiedName)){ - mustArrayLength = 3; - } else { - mustArrayLength = 4; - } - Map[] mustArray = new Map[mustArrayLength]; - Map boolFilter = new HashMap<>(); - Map[] mustNotArray = new Map[2]; - - mustArray[0] = mapOf("term", mapOf("__state", "ACTIVE")); - if(StringUtils.isNotEmpty(glossaryQualifiedName)) { - mustArray[1] = mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossaryTerm", "AtlasGlossaryCategory"))); - mustArray[2] = mapOf("term", mapOf("__glossary", glossaryQualifiedName)); - } else{ - mustArray[1] = mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossary"))); - } - - if(StringUtils.isEmpty(parentQualifiedName)) { - mustNotArray[0] = mapOf("exists", mapOf("field", "__categories")); - mustNotArray[1] = mapOf("exists", mapOf("field", "__parentCategory")); - boolFilter.put("must_not", mustNotArray); - } - else { - Map[] shouldParentArray = new Map[2]; - shouldParentArray[0] = mapOf("term", mapOf("__categories", parentQualifiedName)); - shouldParentArray[1] = mapOf("term", mapOf("__parentCategory", parentQualifiedName)); - mustArray[3] = mapOf("bool",mapOf("should", shouldParentArray)); - } - - boolFilter.put("must", mustArray); - - Map nestedBoolQuery = mapOf("bool", boolFilter); - - Map topBoolFilter = mapOf("filter", nestedBoolQuery); - - boolQuery.put("bool", topBoolFilter); - - return boolQuery; - } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java index 532538aa1b..1ad8752fff 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java @@ -124,15 +124,16 @@ private void processCreateCategory(AtlasEntity entity, AtlasVertex vertex) throw } String glossaryQualifiedName = (String) anchor.getAttribute(QUALIFIED_NAME); + categoryExists(catName, glossaryQualifiedName); + validateParent(glossaryQualifiedName); + if (parentCategory != null) { parentQname = (String) parentCategory.getAttribute(QUALIFIED_NAME); } String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); if(StringUtils.isEmpty(lexicographicalSortOrder)){ - assignNewLexicographicalSortOrder(entity,glossaryQualifiedName, parentQname); + assignNewLexicographicalSortOrder(entity,glossaryQualifiedName, parentQname, this.discovery); } - categoryExists(catName, glossaryQualifiedName); - validateParent(glossaryQualifiedName); entity.setAttribute(QUALIFIED_NAME, createQualifiedName(vertex)); AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, new AtlasEntityHeader(entity)), diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java index 5cea0d0683..fe629d45ce 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java @@ -20,10 +20,12 @@ import org.apache.atlas.AtlasErrorCode; import org.apache.atlas.RequestContext; +import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasStruct; import org.apache.atlas.model.instance.EntityMutations; +import org.apache.atlas.repository.graphdb.AtlasGraph; import org.apache.atlas.repository.graphdb.AtlasVertex; import org.apache.atlas.repository.store.graph.v2.AtlasGraphUtilsV2; import org.apache.atlas.repository.store.graph.v2.EntityGraphRetriever; @@ -37,8 +39,7 @@ import static org.apache.atlas.repository.Constants.NAME; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; -import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.getUUID; -import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.isNameInvalid; +import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.*; import static org.apache.atlas.type.Constants.LEXICOGRAPHICAL_SORT_ORDER; public class GlossaryPreProcessor implements PreProcessor { @@ -46,10 +47,16 @@ public class GlossaryPreProcessor implements PreProcessor { private final AtlasTypeRegistry typeRegistry; private final EntityGraphRetriever entityRetriever; + protected EntityDiscoveryService discovery; - public GlossaryPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever) { + public GlossaryPreProcessor(AtlasTypeRegistry typeRegistry, EntityGraphRetriever entityRetriever, AtlasGraph graph) { this.entityRetriever = entityRetriever; this.typeRegistry = typeRegistry; + try{ + this.discovery = new EntityDiscoveryService(typeRegistry, graph, null, null, null, null); + } catch (Exception e) { + e.printStackTrace(); + } } @Override @@ -79,14 +86,16 @@ private void processCreateGlossary(AtlasStruct entity) throws AtlasBaseException throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); } String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); - // TODO : Figure the placement for this method for glossary creation. -// if(StringUtils.isEmpty(lexicographicalSortOrder)){ -// assignNewLexicographicalSortOrder(entity,null, null); -// } + + if (glossaryExists(glossaryName)) { throw new AtlasBaseException(AtlasErrorCode.GLOSSARY_ALREADY_EXISTS,glossaryName); } + if(StringUtils.isEmpty(lexicographicalSortOrder)){ + assignNewLexicographicalSortOrder((AtlasEntity) entity, null, null, this.discovery); + } + entity.setAttribute(QUALIFIED_NAME, createQualifiedName()); RequestContext.get().endMetricRecord(metricRecorder); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java index 53e12ea93e..e80c739e79 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java @@ -45,11 +45,13 @@ import org.springframework.stereotype.Component; import java.util.Iterator; import java.util.List; +import java.util.Objects; import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.graph.GraphHelper.getActiveParentVertices; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.*; import static org.apache.atlas.repository.store.graph.v2.tasks.MeaningsTaskFactory.UPDATE_ENTITY_MEANINGS_ON_TERM_UPDATE; +import static org.apache.atlas.type.Constants.LEXICOGRAPHICAL_SORT_ORDER; @Component public class TermPreProcessor extends AbstractGlossaryPreProcessor { @@ -95,7 +97,12 @@ private void processCreateTerm(AtlasEntity entity, AtlasVertex vertex) throws At termExists(termName, glossaryQName); - validateCategory(entity); + String parentQname = validateAndGetCategory(entity); + + String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); + if(StringUtils.isEmpty(lexicographicalSortOrder)){ + assignNewLexicographicalSortOrder(entity, glossaryQName, parentQname, this.discovery); + } entity.setAttribute(QUALIFIED_NAME, createQualifiedName()); AtlasAuthorizationUtils.verifyAccess(new AtlasEntityAccessRequest(typeRegistry, AtlasPrivilege.ENTITY_CREATE, new AtlasEntityHeader(entity)), @@ -114,7 +121,7 @@ private void processUpdateTerm(AtlasEntity entity, AtlasVertex vertex) throws At throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); } - validateCategory(entity); + validateAndGetCategory(entity); AtlasEntity storedTerm = entityRetriever.toAtlasEntity(vertex); AtlasRelatedObjectId currentGlossary = (AtlasRelatedObjectId) storedTerm.getRelationshipAttribute(ANCHOR); @@ -159,15 +166,16 @@ private void processUpdateTerm(AtlasEntity entity, AtlasVertex vertex) throws At RequestContext.get().endMetricRecord(metricRecorder); } - private void validateCategory(AtlasEntity entity) throws AtlasBaseException { + private String validateAndGetCategory(AtlasEntity entity) throws AtlasBaseException { String glossaryQualifiedName = (String) anchor.getAttribute(QUALIFIED_NAME); + String categoryQualifiedName = null; + if (entity.hasRelationshipAttribute(ATTR_CATEGORIES) && entity.getRelationshipAttribute(ATTR_CATEGORIES) != null) { List categories = (List) entity.getRelationshipAttribute(ATTR_CATEGORIES); if (CollectionUtils.isNotEmpty(categories)) { AtlasObjectId category = categories.get(0); - String categoryQualifiedName; if (category.getUniqueAttributes() != null && category.getUniqueAttributes().containsKey(QUALIFIED_NAME)) { categoryQualifiedName = (String) category.getUniqueAttributes().get(QUALIFIED_NAME); @@ -181,6 +189,7 @@ private void validateCategory(AtlasEntity entity) throws AtlasBaseException { } } } + return categoryQualifiedName; } public String moveTermToAnotherGlossary(AtlasEntity entity, AtlasVertex vertex, diff --git a/server-api/src/main/java/org/apache/atlas/RequestContext.java b/server-api/src/main/java/org/apache/atlas/RequestContext.java index 565832b7bd..3680d97a08 100644 --- a/server-api/src/main/java/org/apache/atlas/RequestContext.java +++ b/server-api/src/main/java/org/apache/atlas/RequestContext.java @@ -47,6 +47,9 @@ public class RequestContext { private final Map updatedEntities = new HashMap<>(); private final Map deletedEntities = new HashMap<>(); private final Map restoreEntities = new HashMap<>(); + + + private Map lexoRankCache = null; private final Map entityCache = new HashMap<>(); private final Map entityHeaderCache = new HashMap<>(); private final Map entityExtInfoCache = new HashMap<>(); @@ -162,6 +165,7 @@ public void clearCache() { this.requestContextHeaders.clear(); this.relationshipEndToVertexIdMap.clear(); this.relationshipMutationMap.clear(); + this.lexoRankCache = null; this.currentTask = null; this.skipAuthorizationCheck = false; this.delayTagNotifications = false; @@ -788,4 +792,12 @@ public void clearMutationContext(String event) { public Map> getRelationshipMutationMap() { return relationshipMutationMap; } + + public Map getLexoRankCache() { + return lexoRankCache; + } + + public void setLexoRankCache(Map lexoRankCache) { + this.lexoRankCache = lexoRankCache; + } } \ No newline at end of file From 1bcff4ee03af20360025619edbd651d923cfc7a8 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Tue, 28 May 2024 12:48:31 +0530 Subject: [PATCH 03/20] Added LexoRank Validation method, with tentative rebalancing trigger logic --- .../v2/preprocessor/PreProcessorUtils.java | 22 +++++++++++++++++++ .../glossary/CategoryPreProcessor.java | 7 ++++++ .../glossary/GlossaryPreProcessor.java | 8 ++++++- .../glossary/TermPreProcessor.java | 8 +++++++ 4 files changed, 44 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index 111e4b2356..ad954ffc94 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -23,6 +23,8 @@ import org.slf4j.LoggerFactory; import java.util.*; +import java.util.regex.Matcher; +import java.util.regex.Pattern; import static org.apache.atlas.repository.Constants.QUERY_COLLECTION_ENTITY_TYPE; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; @@ -89,6 +91,9 @@ public enum MigrationStatus { public static final String CHILDREN_QUERIES = "__Namespace.childrenQueries"; public static final String CHILDREN_FOLDERS = "__Namespace.childrenFolders"; + public static final int REBALANCING_TRIGGER = 119; + public static final int PRE_DELIMITER_LENGTH = 9; + public static final String LEXORANK_HARD_LIMIT = "" + (256 - PRE_DELIMITER_LENGTH); public static String getUUID(){ return NanoIdUtils.randomNanoId(); @@ -207,6 +212,23 @@ public static void verifyDuplicateAssetByName(String typeName, String assetName, } } + public static void isValidLexoRank(String input) throws AtlasBaseException { + String pattern = "^0\\|[0-9a-z]{6}:(?:[0-9a-z]{0," + LEXORANK_HARD_LIMIT + "})?$"; + + Pattern regex = Pattern.compile(pattern); + + Matcher matcher = regex.matcher(input); + + if(!matcher.matches()){ + throw new AtlasBaseException("Invalid LexicographicSortOrder"); + } + // TODO : Add the rebalancing logic here + int colonIndex = input.indexOf(":"); + if (colonIndex != -1 && input.substring(colonIndex + 1).length() >= REBALANCING_TRIGGER) { + // Rebalancing trigger + } + } + public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String glossaryQualifiedName, String parentQualifiedName, EntityDiscoveryService discovery) { Map lexoRankCache = RequestContext.get().getLexoRankCache(); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java index 1ad8752fff..de11636a82 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java @@ -133,6 +133,8 @@ private void processCreateCategory(AtlasEntity entity, AtlasVertex vertex) throw String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); if(StringUtils.isEmpty(lexicographicalSortOrder)){ assignNewLexicographicalSortOrder(entity,glossaryQualifiedName, parentQname, this.discovery); + } else { + isValidLexoRank(lexicographicalSortOrder); } entity.setAttribute(QUALIFIED_NAME, createQualifiedName(vertex)); @@ -160,6 +162,11 @@ private void processUpdateCategory(AtlasEntity entity, AtlasVertex vertex) throw String newGlossaryQualifiedName = (String) anchor.getAttribute(QUALIFIED_NAME); + String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); + if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { + isValidLexoRank(lexicographicalSortOrder); + } + if (!currentGlossaryQualifiedName.equals(newGlossaryQualifiedName)){ //Auth check isAuthorized(currentGlossaryHeader, anchor); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java index fe629d45ce..d74bc54528 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java @@ -92,8 +92,10 @@ private void processCreateGlossary(AtlasStruct entity) throws AtlasBaseException throw new AtlasBaseException(AtlasErrorCode.GLOSSARY_ALREADY_EXISTS,glossaryName); } - if(StringUtils.isEmpty(lexicographicalSortOrder)){ + if(StringUtils.isEmpty(lexicographicalSortOrder)) { assignNewLexicographicalSortOrder((AtlasEntity) entity, null, null, this.discovery); + } else { + isValidLexoRank(lexicographicalSortOrder); } entity.setAttribute(QUALIFIED_NAME, createQualifiedName()); @@ -112,6 +114,10 @@ private void processUpdateGlossary(AtlasStruct entity, AtlasVertex vertex) throw if (StringUtils.isEmpty(glossaryName) || isNameInvalid(glossaryName)) { throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); } + String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); + if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { + isValidLexoRank(lexicographicalSortOrder); + } String vertexQnName = vertex.getProperty(QUALIFIED_NAME, String.class); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java index e80c739e79..36528f0fe7 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java @@ -95,6 +95,7 @@ private void processCreateTerm(AtlasEntity entity, AtlasVertex vertex) throws At String glossaryQName = (String) anchor.getAttribute(QUALIFIED_NAME); + termExists(termName, glossaryQName); String parentQname = validateAndGetCategory(entity); @@ -102,6 +103,8 @@ private void processCreateTerm(AtlasEntity entity, AtlasVertex vertex) throws At String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); if(StringUtils.isEmpty(lexicographicalSortOrder)){ assignNewLexicographicalSortOrder(entity, glossaryQName, parentQname, this.discovery); + } else { + isValidLexoRank(lexicographicalSortOrder); } entity.setAttribute(QUALIFIED_NAME, createQualifiedName()); @@ -132,6 +135,11 @@ private void processUpdateTerm(AtlasEntity entity, AtlasVertex vertex) throws At String newGlossaryQualifiedName = (String) anchor.getAttribute(QUALIFIED_NAME); + String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); + if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { + isValidLexoRank(lexicographicalSortOrder); + } + if (!currentGlossaryQualifiedName.equals(newGlossaryQualifiedName)){ //Auth check isAuthorized(currentGlossaryHeader, anchor); From f4a38dcab4bf9b748f52134399533719193ff7a4 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Tue, 28 May 2024 18:00:28 +0530 Subject: [PATCH 04/20] Corrected a method rename --- .../store/graph/v2/preprocessor/glossary/TermPreProcessor.java | 1 - 1 file changed, 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java index 36528f0fe7..39e8654437 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java @@ -176,7 +176,6 @@ private void processUpdateTerm(AtlasEntity entity, AtlasVertex vertex) throws At private String validateAndGetCategory(AtlasEntity entity) throws AtlasBaseException { String glossaryQualifiedName = (String) anchor.getAttribute(QUALIFIED_NAME); - String categoryQualifiedName = null; if (entity.hasRelationshipAttribute(ATTR_CATEGORIES) && entity.getRelationshipAttribute(ATTR_CATEGORIES) != null) { From 8c1c21ae32fde932f1b56084aa0cd851e9981300 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Wed, 29 May 2024 01:05:33 +0530 Subject: [PATCH 05/20] Added a check in validateLexoRank which checks if another entity with duplicate lexorank doesnt exist in scenario where lexorank is being sent from user. --- .../v2/preprocessor/PreProcessorUtils.java | 80 ++++++++++++++++++- .../glossary/CategoryPreProcessor.java | 5 +- .../glossary/GlossaryPreProcessor.java | 4 +- .../glossary/TermPreProcessor.java | 6 +- 4 files changed, 87 insertions(+), 8 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index ad954ffc94..31d3f74c30 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -212,7 +212,7 @@ public static void verifyDuplicateAssetByName(String typeName, String assetName, } } - public static void isValidLexoRank(String input) throws AtlasBaseException { + public static void isValidLexoRank(String input, String glossaryQualifiedName, String parentQualifiedName, EntityDiscoveryService discovery) throws AtlasBaseException { String pattern = "^0\\|[0-9a-z]{6}:(?:[0-9a-z]{0," + LEXORANK_HARD_LIMIT + "})?$"; Pattern regex = Pattern.compile(pattern); @@ -222,6 +222,21 @@ public static void isValidLexoRank(String input) throws AtlasBaseException { if(!matcher.matches()){ throw new AtlasBaseException("Invalid LexicographicSortOrder"); } + + Map dslQuery = createDSLforCheckingPreExistingLexoRank(input, glossaryQualifiedName, parentQualifiedName); + List categories = new ArrayList<>(); + try { + IndexSearchParams searchParams = new IndexSearchParams(); + searchParams.setAttributes(new HashSet<>()); + searchParams.setDsl(dslQuery); + categories = discovery.directIndexSearch(searchParams).getEntities(); + } catch (AtlasBaseException e) { + e.printStackTrace(); + } + + if(!CollectionUtils.isEmpty(categories)){ + throw new AtlasBaseException("Invalid LexicographicSortOrder"); + } // TODO : Add the rebalancing logic here int colonIndex = input.indexOf(":"); if (colonIndex != -1 && input.substring(colonIndex + 1).length() >= REBALANCING_TRIGGER) { @@ -278,7 +293,70 @@ public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String RequestContext.get().setLexoRankCache(lexoRankCache); } + public static Map createDSLforCheckingPreExistingLexoRank(String lexoRank, String glossaryQualifiedName, String parentQualifiedName) { + + Map sortKeyOrder = mapOf(LEXICOGRAPHICAL_SORT_ORDER, mapOf("order", "desc")); + Map scoreSortOrder = mapOf("_score", mapOf("order", "desc")); + Map displayNameSortOrder = mapOf("displayName.keyword", mapOf("order", "desc")); + + Object[] sortArray = {sortKeyOrder, scoreSortOrder, displayNameSortOrder}; + + Map functionScore = mapOf("query", buildBoolQueryDuplicateLexoRank(lexoRank, glossaryQualifiedName, parentQualifiedName)); + + Map dsl = new HashMap<>(); + dsl.put("from", 0); + dsl.put("size", 100); + dsl.put("sort", sortArray); + dsl.put("query", mapOf("function_score", functionScore)); + + return dsl; + } + + private static Map buildBoolQueryDuplicateLexoRank(String lexoRank, String glossaryQualifiedName, String parentQualifiedName) { + Map boolQuery = new HashMap<>(); + int mustArrayLength = 0; + if(StringUtils.isEmpty(parentQualifiedName) && StringUtils.isEmpty(glossaryQualifiedName)){ + mustArrayLength = 3; + } else if(StringUtils.isEmpty(parentQualifiedName) && StringUtils.isNotEmpty(glossaryQualifiedName)){ + mustArrayLength = 4; + } else { + mustArrayLength = 5; + } + Map[] mustArray = new Map[mustArrayLength]; + Map boolFilter = new HashMap<>(); + Map[] mustNotArray = new Map[2]; + + mustArray[0] = mapOf("term", mapOf("__state", "ACTIVE")); + mustArray[1] = mapOf("term", mapOf(LEXICOGRAPHICAL_SORT_ORDER, lexoRank)); + if(StringUtils.isNotEmpty(glossaryQualifiedName)) { + mustArray[2] = mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossaryTerm", "AtlasGlossaryCategory"))); + mustArray[3] = mapOf("term", mapOf("__glossary", glossaryQualifiedName)); + } else{ + mustArray[2] = mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossary"))); + } + if(StringUtils.isEmpty(parentQualifiedName)) { + mustNotArray[0] = mapOf("exists", mapOf("field", "__categories")); + mustNotArray[1] = mapOf("exists", mapOf("field", "__parentCategory")); + boolFilter.put("must_not", mustNotArray); + } + else { + Map[] shouldParentArray = new Map[2]; + shouldParentArray[0] = mapOf("term", mapOf("__categories", parentQualifiedName)); + shouldParentArray[1] = mapOf("term", mapOf("__parentCategory", parentQualifiedName)); + mustArray[4] = mapOf("bool",mapOf("should", shouldParentArray)); + } + + boolFilter.put("must", mustArray); + + Map nestedBoolQuery = mapOf("bool", boolFilter); + + Map topBoolFilter = mapOf("filter", nestedBoolQuery); + + boolQuery.put("bool", topBoolFilter); + + return boolQuery; + } public static Map generateDSLQueryForLastCategory(String glossaryQualifiedName, String parentQualifiedName) { Map sortKeyOrder = mapOf(LEXICOGRAPHICAL_SORT_ORDER, mapOf("order", "desc")); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java index de11636a82..0f90531561 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java @@ -134,7 +134,7 @@ private void processCreateCategory(AtlasEntity entity, AtlasVertex vertex) throw if(StringUtils.isEmpty(lexicographicalSortOrder)){ assignNewLexicographicalSortOrder(entity,glossaryQualifiedName, parentQname, this.discovery); } else { - isValidLexoRank(lexicographicalSortOrder); + isValidLexoRank(lexicographicalSortOrder, glossaryQualifiedName, parentQname, this.discovery); } entity.setAttribute(QUALIFIED_NAME, createQualifiedName(vertex)); @@ -163,8 +163,9 @@ private void processUpdateCategory(AtlasEntity entity, AtlasVertex vertex) throw String newGlossaryQualifiedName = (String) anchor.getAttribute(QUALIFIED_NAME); String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); + String parentQname = (String) parentCategory.getAttribute(QUALIFIED_NAME); if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { - isValidLexoRank(lexicographicalSortOrder); + isValidLexoRank(lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery); } if (!currentGlossaryQualifiedName.equals(newGlossaryQualifiedName)){ diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java index d74bc54528..b585705284 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java @@ -95,7 +95,7 @@ private void processCreateGlossary(AtlasStruct entity) throws AtlasBaseException if(StringUtils.isEmpty(lexicographicalSortOrder)) { assignNewLexicographicalSortOrder((AtlasEntity) entity, null, null, this.discovery); } else { - isValidLexoRank(lexicographicalSortOrder); + isValidLexoRank(lexicographicalSortOrder, "", "", this.discovery); } entity.setAttribute(QUALIFIED_NAME, createQualifiedName()); @@ -116,7 +116,7 @@ private void processUpdateGlossary(AtlasStruct entity, AtlasVertex vertex) throw } String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { - isValidLexoRank(lexicographicalSortOrder); + isValidLexoRank(lexicographicalSortOrder, "", "", this.discovery); } String vertexQnName = vertex.getProperty(QUALIFIED_NAME, String.class); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java index 39e8654437..9af03cabf8 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java @@ -104,7 +104,7 @@ private void processCreateTerm(AtlasEntity entity, AtlasVertex vertex) throws At if(StringUtils.isEmpty(lexicographicalSortOrder)){ assignNewLexicographicalSortOrder(entity, glossaryQName, parentQname, this.discovery); } else { - isValidLexoRank(lexicographicalSortOrder); + isValidLexoRank(lexicographicalSortOrder, glossaryQName, parentQname, this.discovery); } entity.setAttribute(QUALIFIED_NAME, createQualifiedName()); @@ -124,7 +124,7 @@ private void processUpdateTerm(AtlasEntity entity, AtlasVertex vertex) throws At throw new AtlasBaseException(AtlasErrorCode.INVALID_DISPLAY_NAME); } - validateAndGetCategory(entity); + String parentQname = validateAndGetCategory(entity); AtlasEntity storedTerm = entityRetriever.toAtlasEntity(vertex); AtlasRelatedObjectId currentGlossary = (AtlasRelatedObjectId) storedTerm.getRelationshipAttribute(ANCHOR); @@ -137,7 +137,7 @@ private void processUpdateTerm(AtlasEntity entity, AtlasVertex vertex) throws At String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { - isValidLexoRank(lexicographicalSortOrder); + isValidLexoRank(lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery); } if (!currentGlossaryQualifiedName.equals(newGlossaryQualifiedName)){ From 30b4d47e86b04ae910c613e3a97d6f5a6eb685dc Mon Sep 17 00:00:00 2001 From: hr2904 Date: Wed, 29 May 2024 17:15:35 +0530 Subject: [PATCH 06/20] Added a check , such that if bulk request is from migration, it will override the check for for duplicate lexorank. --- .../v2/preprocessor/PreProcessorUtils.java | 30 ++++++++++--------- .../glossary/CategoryPreProcessor.java | 5 +++- 2 files changed, 20 insertions(+), 15 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index 31d3f74c30..e131fcedcb 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -214,7 +214,8 @@ public static void verifyDuplicateAssetByName(String typeName, String assetName, public static void isValidLexoRank(String input, String glossaryQualifiedName, String parentQualifiedName, EntityDiscoveryService discovery) throws AtlasBaseException { String pattern = "^0\\|[0-9a-z]{6}:(?:[0-9a-z]{0," + LEXORANK_HARD_LIMIT + "})?$"; - + // TODO : To remove this after migration is successful on all tenants and custom-sort is successfully GA + Boolean requestFromMigration = RequestContext.get().getRequestContextHeaders().getOrDefault("x-atlan-request-id", "").contains("custom-sort-migration"); Pattern regex = Pattern.compile(pattern); Matcher matcher = regex.matcher(input); @@ -222,20 +223,21 @@ public static void isValidLexoRank(String input, String glossaryQualifiedName, S if(!matcher.matches()){ throw new AtlasBaseException("Invalid LexicographicSortOrder"); } + if(!requestFromMigration) { + Map dslQuery = createDSLforCheckingPreExistingLexoRank(input, glossaryQualifiedName, parentQualifiedName); + List categories = new ArrayList<>(); + try { + IndexSearchParams searchParams = new IndexSearchParams(); + searchParams.setAttributes(new HashSet<>()); + searchParams.setDsl(dslQuery); + categories = discovery.directIndexSearch(searchParams).getEntities(); + } catch (AtlasBaseException e) { + e.printStackTrace(); + } - Map dslQuery = createDSLforCheckingPreExistingLexoRank(input, glossaryQualifiedName, parentQualifiedName); - List categories = new ArrayList<>(); - try { - IndexSearchParams searchParams = new IndexSearchParams(); - searchParams.setAttributes(new HashSet<>()); - searchParams.setDsl(dslQuery); - categories = discovery.directIndexSearch(searchParams).getEntities(); - } catch (AtlasBaseException e) { - e.printStackTrace(); - } - - if(!CollectionUtils.isEmpty(categories)){ - throw new AtlasBaseException("Invalid LexicographicSortOrder"); + if (!CollectionUtils.isEmpty(categories)) { + throw new AtlasBaseException("Invalid LexicographicSortOrder"); + } } // TODO : Add the rebalancing logic here int colonIndex = input.indexOf(":"); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java index 0f90531561..baac9c82c3 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java @@ -163,7 +163,10 @@ private void processUpdateCategory(AtlasEntity entity, AtlasVertex vertex) throw String newGlossaryQualifiedName = (String) anchor.getAttribute(QUALIFIED_NAME); String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); - String parentQname = (String) parentCategory.getAttribute(QUALIFIED_NAME); + String parentQname = ""; + if(Objects.nonNull(parentCategory)) { + parentQname = (String) parentCategory.getAttribute(QUALIFIED_NAME); + } if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { isValidLexoRank(lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery); } From 5d06cba8e3f7f544061df50344a75863359d578d Mon Sep 17 00:00:00 2001 From: hr2904 Date: Mon, 3 Jun 2024 00:00:21 +0530 Subject: [PATCH 07/20] Modified ES query. --- .../org/apache/atlas/glossary/GlossaryUtils.java | 4 ++-- .../graph/v2/preprocessor/PreProcessorUtils.java | 13 ++++++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/glossary/GlossaryUtils.java b/repository/src/main/java/org/apache/atlas/glossary/GlossaryUtils.java index 0f16d0dc82..2310726b2e 100644 --- a/repository/src/main/java/org/apache/atlas/glossary/GlossaryUtils.java +++ b/repository/src/main/java/org/apache/atlas/glossary/GlossaryUtils.java @@ -55,8 +55,8 @@ public abstract class GlossaryUtils { public static final String TERM_ASSIGNMENT_ATTR_SOURCE = "source"; static final String ATLAS_GLOSSARY_TYPENAME = "AtlasGlossary"; - static final String ATLAS_GLOSSARY_TERM_TYPENAME = "AtlasGlossaryTerm"; - static final String ATLAS_GLOSSARY_CATEGORY_TYPENAME = "AtlasGlossaryCategory"; + public static final String ATLAS_GLOSSARY_TERM_TYPENAME = "AtlasGlossaryTerm"; + public static final String ATLAS_GLOSSARY_CATEGORY_TYPENAME = "AtlasGlossaryCategory"; public static final String NAME = "name"; public static final String QUALIFIED_NAME = "qualifiedName"; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index e131fcedcb..6e1f559ad9 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -26,6 +26,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import static org.apache.atlas.glossary.GlossaryUtils.ATLAS_GLOSSARY_TERM_TYPENAME; import static org.apache.atlas.repository.Constants.QUERY_COLLECTION_ENTITY_TYPE; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY; @@ -262,7 +263,8 @@ public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String Set attributes = new HashSet<>(); attributes.add(LEXICOGRAPHICAL_SORT_ORDER); List categories = null; - Map dslQuery = generateDSLQueryForLastCategory(glossaryQualifiedName, parentQualifiedName); + boolean isTerm = entity.getTypeName().equals(ATLAS_GLOSSARY_TERM_TYPENAME) ? true : false; + Map dslQuery = generateDSLQueryForLastCategory(glossaryQualifiedName, parentQualifiedName, isTerm); try { IndexSearchParams searchParams = new IndexSearchParams(); searchParams.setAttributes(attributes); @@ -359,7 +361,7 @@ private static Map buildBoolQueryDuplicateLexoRank(String lexoRa return boolQuery; } - public static Map generateDSLQueryForLastCategory(String glossaryQualifiedName, String parentQualifiedName) { + public static Map generateDSLQueryForLastCategory(String glossaryQualifiedName, String parentQualifiedName, boolean isTerm) { Map sortKeyOrder = mapOf(LEXICOGRAPHICAL_SORT_ORDER, mapOf("order", "desc")); Map scoreSortOrder = mapOf("_score", mapOf("order", "desc")); @@ -367,7 +369,7 @@ public static Map generateDSLQueryForLastCategory(String glossar Object[] sortArray = {sortKeyOrder, scoreSortOrder, displayNameSortOrder}; - Map functionScore = mapOf("query", buildBoolQuery(glossaryQualifiedName, parentQualifiedName)); + Map functionScore = mapOf("query", buildBoolQuery(glossaryQualifiedName, parentQualifiedName, isTerm)); Map dsl = new HashMap<>(); dsl.put("from", 0); @@ -378,7 +380,7 @@ public static Map generateDSLQueryForLastCategory(String glossar return dsl; } - private static Map buildBoolQuery(String glossaryQualifiedName, String parentQualifiedName) { + private static Map buildBoolQuery(String glossaryQualifiedName, String parentQualifiedName, boolean isTerm) { Map boolQuery = new HashMap<>(); int mustArrayLength = 0; if(StringUtils.isEmpty(parentQualifiedName) && StringUtils.isEmpty(glossaryQualifiedName)){ @@ -394,7 +396,8 @@ private static Map buildBoolQuery(String glossaryQualifiedName, mustArray[0] = mapOf("term", mapOf("__state", "ACTIVE")); if(StringUtils.isNotEmpty(glossaryQualifiedName)) { - mustArray[1] = mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossaryTerm", "AtlasGlossaryCategory"))); + String typeName = isTerm ? "AtlasGlossaryTerm" : "AtlasGlossaryCategory"; + mustArray[1] = mapOf("term", mapOf("__typeName.keyword", typeName)); mustArray[2] = mapOf("term", mapOf("__glossary", glossaryQualifiedName)); } else{ mustArray[1] = mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossary"))); From 378859fdbcf136914923f9cc8e0b3ffe4d670b21 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Mon, 3 Jun 2024 11:53:17 +0530 Subject: [PATCH 08/20] Modified an edge statement such that, when a new lexorank for cat is generated, it is not a duplicate of a term in same linear data set --- .../graph/v2/preprocessor/PreProcessorUtils.java | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index 6e1f559ad9..3453077b24 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -278,7 +278,7 @@ public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String for (AtlasEntityHeader category : categories) { String lexicographicalSortOrder = (String) category.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); if (StringUtils.isNotEmpty(lexicographicalSortOrder)) { - lastLexoRank = lexicographicalSortOrder; + lastLexoRank = getValidLexorank(lexicographicalSortOrder, glossaryQualifiedName, parentQualifiedName, discovery); } else { lastLexoRank = INIT_LEXORANK_OFFSET; } @@ -297,6 +297,17 @@ public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String RequestContext.get().setLexoRankCache(lexoRankCache); } + private static String getValidLexorank(String lastLexoRank, String glossaryQualifiedName, String parentQualifiedName, EntityDiscoveryService discovery) { + LexoRank parsedLexoRank = LexoRank.parse(lastLexoRank); + LexoRank nextLexoRank = parsedLexoRank.genNext().genNext(); + try { + isValidLexoRank(nextLexoRank.toString(), glossaryQualifiedName, parentQualifiedName, discovery); + return lastLexoRank; + } catch (AtlasBaseException e){ + return parsedLexoRank.between(nextLexoRank).genPrev().genPrev().toString(); + } + } + public static Map createDSLforCheckingPreExistingLexoRank(String lexoRank, String glossaryQualifiedName, String parentQualifiedName) { Map sortKeyOrder = mapOf(LEXICOGRAPHICAL_SORT_ORDER, mapOf("order", "desc")); From ffd3e7345f01d1be54accea41a7fab18f3e3973b Mon Sep 17 00:00:00 2001 From: hr2904 Date: Mon, 3 Jun 2024 12:53:35 +0530 Subject: [PATCH 09/20] Reverted the check, instead added padded new offset for terms. --- .../v2/preprocessor/PreProcessorUtils.java | 56 +++++++------------ 1 file changed, 20 insertions(+), 36 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index 3453077b24..b4dd54abfc 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -17,6 +17,7 @@ import org.apache.atlas.util.NanoIdUtils; import org.apache.atlas.util.lexoRank.LexoRank; import org.apache.atlas.utils.AtlasEntityUtil; +import org.apache.atlas.v1.model.instance.Id; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; @@ -30,6 +31,8 @@ import static org.apache.atlas.repository.Constants.QUERY_COLLECTION_ENTITY_TYPE; import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_IS_ENABLED; +import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_SERVICE_NAME; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; import static org.apache.atlas.type.Constants.LEXICOGRAPHICAL_SORT_ORDER; @@ -46,6 +49,7 @@ public class PreProcessorUtils { public static final String GLOSSARY_TERM_REL_TYPE = "AtlasGlossaryTermAnchor"; public static final String GLOSSARY_CATEGORY_REL_TYPE = "AtlasGlossaryCategoryAnchor"; public static final String INIT_LEXORANK_OFFSET = "0|100000:"; + public static final String INIT_TERM_LEXORANK_OFFSET = "0|500000:"; //DataMesh models constants public static final String PARENT_DOMAIN_REL_TYPE = "parentDomain"; @@ -278,13 +282,13 @@ public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String for (AtlasEntityHeader category : categories) { String lexicographicalSortOrder = (String) category.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); if (StringUtils.isNotEmpty(lexicographicalSortOrder)) { - lastLexoRank = getValidLexorank(lexicographicalSortOrder, glossaryQualifiedName, parentQualifiedName, discovery); + lastLexoRank = lexicographicalSortOrder; } else { - lastLexoRank = INIT_LEXORANK_OFFSET; + lastLexoRank = isTerm ? INIT_TERM_LEXORANK_OFFSET : INIT_LEXORANK_OFFSET; } } } else { - lastLexoRank = INIT_LEXORANK_OFFSET; + lastLexoRank = isTerm ? INIT_TERM_LEXORANK_OFFSET : INIT_LEXORANK_OFFSET; } } @@ -297,17 +301,6 @@ public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String RequestContext.get().setLexoRankCache(lexoRankCache); } - private static String getValidLexorank(String lastLexoRank, String glossaryQualifiedName, String parentQualifiedName, EntityDiscoveryService discovery) { - LexoRank parsedLexoRank = LexoRank.parse(lastLexoRank); - LexoRank nextLexoRank = parsedLexoRank.genNext().genNext(); - try { - isValidLexoRank(nextLexoRank.toString(), glossaryQualifiedName, parentQualifiedName, discovery); - return lastLexoRank; - } catch (AtlasBaseException e){ - return parsedLexoRank.between(nextLexoRank).genPrev().genPrev().toString(); - } - } - public static Map createDSLforCheckingPreExistingLexoRank(String lexoRank, String glossaryQualifiedName, String parentQualifiedName) { Map sortKeyOrder = mapOf(LEXICOGRAPHICAL_SORT_ORDER, mapOf("order", "desc")); @@ -329,37 +322,28 @@ public static Map createDSLforCheckingPreExistingLexoRank(String private static Map buildBoolQueryDuplicateLexoRank(String lexoRank, String glossaryQualifiedName, String parentQualifiedName) { Map boolQuery = new HashMap<>(); - int mustArrayLength = 0; - if(StringUtils.isEmpty(parentQualifiedName) && StringUtils.isEmpty(glossaryQualifiedName)){ - mustArrayLength = 3; - } else if(StringUtils.isEmpty(parentQualifiedName) && StringUtils.isNotEmpty(glossaryQualifiedName)){ - mustArrayLength = 4; - } else { - mustArrayLength = 5; - } - Map[] mustArray = new Map[mustArrayLength]; Map boolFilter = new HashMap<>(); - Map[] mustNotArray = new Map[2]; - - mustArray[0] = mapOf("term", mapOf("__state", "ACTIVE")); - mustArray[1] = mapOf("term", mapOf(LEXICOGRAPHICAL_SORT_ORDER, lexoRank)); + List> mustArray = new ArrayList<>(); + List> mustNotArray = new ArrayList<>(); + mustArray.add(mapOf("term", mapOf("__state", "ACTIVE"))); + mustArray.add(mapOf("term", mapOf(LEXICOGRAPHICAL_SORT_ORDER, lexoRank))); if(StringUtils.isNotEmpty(glossaryQualifiedName)) { - mustArray[2] = mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossaryTerm", "AtlasGlossaryCategory"))); - mustArray[3] = mapOf("term", mapOf("__glossary", glossaryQualifiedName)); + mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossaryTerm", "AtlasGlossaryCategory")))); + mustArray.add(mapOf("term", mapOf("__glossary", glossaryQualifiedName))); } else{ - mustArray[2] = mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossary"))); + mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossary")))); } if(StringUtils.isEmpty(parentQualifiedName)) { - mustNotArray[0] = mapOf("exists", mapOf("field", "__categories")); - mustNotArray[1] = mapOf("exists", mapOf("field", "__parentCategory")); + mustNotArray.add(mapOf("exists", mapOf("field", "__categories"))); + mustNotArray.add(mapOf("exists", mapOf("field", "__parentCategory"))); boolFilter.put("must_not", mustNotArray); } else { - Map[] shouldParentArray = new Map[2]; - shouldParentArray[0] = mapOf("term", mapOf("__categories", parentQualifiedName)); - shouldParentArray[1] = mapOf("term", mapOf("__parentCategory", parentQualifiedName)); - mustArray[4] = mapOf("bool",mapOf("should", shouldParentArray)); + List> shouldParentArray = new ArrayList<>(); + shouldParentArray.add(mapOf("term", mapOf("__categories", parentQualifiedName))); + shouldParentArray.add(mapOf("term", mapOf("__parentCategory", parentQualifiedName))); + mustArray.add(mapOf("bool",mapOf("should", shouldParentArray))); } boolFilter.put("must", mustArray); From de5965cfbf795e91f8436d1391939dceba8031c7 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Tue, 4 Jun 2024 11:29:21 +0530 Subject: [PATCH 10/20] fixed the caching logic, by adding the bifurcation for terms and categories --- .../store/graph/v2/preprocessor/PreProcessorUtils.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index b4dd54abfc..701e7d2a35 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -259,15 +259,15 @@ public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String } String lexoRank = ""; String lastLexoRank = ""; + boolean isTerm = entity.getTypeName().equals(ATLAS_GLOSSARY_TERM_TYPENAME) ? true : false; - if(lexoRankCache.containsKey(glossaryQualifiedName + "-" + parentQualifiedName)) { - lastLexoRank = lexoRankCache.get(glossaryQualifiedName + "-" + parentQualifiedName); + if(lexoRankCache.containsKey(entity.getTypeName() + "-" + glossaryQualifiedName + "-" + parentQualifiedName)) { + lastLexoRank = lexoRankCache.get(entity.getTypeName() + "-" + glossaryQualifiedName + "-" + parentQualifiedName); } else { Set attributes = new HashSet<>(); attributes.add(LEXICOGRAPHICAL_SORT_ORDER); List categories = null; - boolean isTerm = entity.getTypeName().equals(ATLAS_GLOSSARY_TERM_TYPENAME) ? true : false; Map dslQuery = generateDSLQueryForLastCategory(glossaryQualifiedName, parentQualifiedName, isTerm); try { IndexSearchParams searchParams = new IndexSearchParams(); @@ -297,7 +297,7 @@ public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String lexoRank = nextLexoRank.toString(); entity.setAttribute(LEXICOGRAPHICAL_SORT_ORDER, lexoRank); - lexoRankCache.put(glossaryQualifiedName + "-" + parentQualifiedName, lexoRank); + lexoRankCache.put(entity.getTypeName() + "-" + glossaryQualifiedName + "-" + parentQualifiedName, lexoRank); RequestContext.get().setLexoRankCache(lexoRankCache); } From 7137d99dc313ba7ba81e107d9dff44f3bbe9a541 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Wed, 12 Jun 2024 18:26:50 +0530 Subject: [PATCH 11/20] Fixed PR comments --- .../v2/preprocessor/PreProcessorUtils.java | 161 ++++++++---------- .../glossary/CategoryPreProcessor.java | 4 +- .../glossary/GlossaryPreProcessor.java | 4 +- .../glossary/TermPreProcessor.java | 4 +- .../lexoRank/system/LexoNumeralSystem10.java | 32 ---- 5 files changed, 76 insertions(+), 129 deletions(-) delete mode 100644 repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem10.java diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index 701e7d2a35..98468aba76 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -17,7 +17,6 @@ import org.apache.atlas.util.NanoIdUtils; import org.apache.atlas.util.lexoRank.LexoRank; import org.apache.atlas.utils.AtlasEntityUtil; -import org.apache.atlas.v1.model.instance.Id; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; @@ -27,12 +26,9 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; +import static org.apache.atlas.glossary.GlossaryUtils.ATLAS_GLOSSARY_CATEGORY_TYPENAME; import static org.apache.atlas.glossary.GlossaryUtils.ATLAS_GLOSSARY_TERM_TYPENAME; -import static org.apache.atlas.repository.Constants.QUERY_COLLECTION_ENTITY_TYPE; -import static org.apache.atlas.repository.Constants.QUALIFIED_NAME; -import static org.apache.atlas.repository.Constants.ENTITY_TYPE_PROPERTY_KEY; -import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_IS_ENABLED; -import static org.apache.atlas.repository.util.AccessControlUtils.ATTR_POLICY_SERVICE_NAME; +import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; import static org.apache.atlas.type.Constants.LEXICOGRAPHICAL_SORT_ORDER; @@ -99,6 +95,13 @@ public enum MigrationStatus { public static final int REBALANCING_TRIGGER = 119; public static final int PRE_DELIMITER_LENGTH = 9; public static final String LEXORANK_HARD_LIMIT = "" + (256 - PRE_DELIMITER_LENGTH); + public static final String LEXORANK_VALID_PATTERN = "^0\\|[0-9a-z]{6}:(?:[0-9a-z]{0," + LEXORANK_HARD_LIMIT + "})?$"; + public static final Set ATTRIBUTES; + static { + Set temp = new HashSet<>(); + temp.add(LEXICOGRAPHICAL_SORT_ORDER); + ATTRIBUTES = Collections.unmodifiableSet(temp); + } public static String getUUID(){ return NanoIdUtils.randomNanoId(); @@ -217,33 +220,35 @@ public static void verifyDuplicateAssetByName(String typeName, String assetName, } } - public static void isValidLexoRank(String input, String glossaryQualifiedName, String parentQualifiedName, EntityDiscoveryService discovery) throws AtlasBaseException { - String pattern = "^0\\|[0-9a-z]{6}:(?:[0-9a-z]{0," + LEXORANK_HARD_LIMIT + "})?$"; + public static void isValidLexoRank(boolean isTerm, String input, String glossaryQualifiedName, String parentQualifiedName, EntityDiscoveryService discovery) throws AtlasBaseException { // TODO : To remove this after migration is successful on all tenants and custom-sort is successfully GA - Boolean requestFromMigration = RequestContext.get().getRequestContextHeaders().getOrDefault("x-atlan-request-id", "").contains("custom-sort-migration"); - Pattern regex = Pattern.compile(pattern); + Pattern regex = Pattern.compile(LEXORANK_VALID_PATTERN); Matcher matcher = regex.matcher(input); if(!matcher.matches()){ - throw new AtlasBaseException("Invalid LexicographicSortOrder"); + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Invalid value for attribute"); + } + Boolean requestFromMigration = RequestContext.get().getRequestContextHeaders().getOrDefault("x-atlan-request-id", "").contains("custom-sort-migration"); + if(requestFromMigration) { + return; + } + Map dslQuery = createDSLforCheckingPreExistingLexoRank(isTerm, input, glossaryQualifiedName, parentQualifiedName); + List assetsWithDuplicateRank = new ArrayList<>(); + try { + IndexSearchParams searchParams = new IndexSearchParams(); + searchParams.setAttributes(new HashSet<>()); + searchParams.setDsl(dslQuery); + assetsWithDuplicateRank = discovery.directIndexSearch(searchParams).getEntities(); + } catch (AtlasBaseException e) { + LOG.error("IndexSearch Error Occured : " + e.getMessage()); + new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Something went wrong with IndexSearch"); } - if(!requestFromMigration) { - Map dslQuery = createDSLforCheckingPreExistingLexoRank(input, glossaryQualifiedName, parentQualifiedName); - List categories = new ArrayList<>(); - try { - IndexSearchParams searchParams = new IndexSearchParams(); - searchParams.setAttributes(new HashSet<>()); - searchParams.setDsl(dslQuery); - categories = discovery.directIndexSearch(searchParams).getEntities(); - } catch (AtlasBaseException e) { - e.printStackTrace(); - } - if (!CollectionUtils.isEmpty(categories)) { - throw new AtlasBaseException("Invalid LexicographicSortOrder"); - } + if (!CollectionUtils.isEmpty(assetsWithDuplicateRank)) { + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Duplicate Lexorank found"); } + // TODO : Add the rebalancing logic here int colonIndex = input.indexOf(":"); if (colonIndex != -1 && input.substring(colonIndex + 1).length() >= REBALANCING_TRIGGER) { @@ -259,19 +264,18 @@ public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String } String lexoRank = ""; String lastLexoRank = ""; - boolean isTerm = entity.getTypeName().equals(ATLAS_GLOSSARY_TERM_TYPENAME) ? true : false; - - if(lexoRankCache.containsKey(entity.getTypeName() + "-" + glossaryQualifiedName + "-" + parentQualifiedName)) { - lastLexoRank = lexoRankCache.get(entity.getTypeName() + "-" + glossaryQualifiedName + "-" + parentQualifiedName); + boolean isTerm = entity.getTypeName().equals(ATLAS_GLOSSARY_TERM_TYPENAME); + String cacheKey = entity.getTypeName() + "-" + glossaryQualifiedName + "-" + parentQualifiedName; + if(lexoRankCache.containsKey(cacheKey)) { + lastLexoRank = lexoRankCache.get(cacheKey); } else { - Set attributes = new HashSet<>(); - attributes.add(LEXICOGRAPHICAL_SORT_ORDER); + List categories = null; - Map dslQuery = generateDSLQueryForLastCategory(glossaryQualifiedName, parentQualifiedName, isTerm); + Map dslQuery = generateDSLQueryForLastChild(glossaryQualifiedName, parentQualifiedName, isTerm); try { IndexSearchParams searchParams = new IndexSearchParams(); - searchParams.setAttributes(attributes); + searchParams.setAttributes(ATTRIBUTES); searchParams.setDsl(dslQuery); categories = discovery.directIndexSearch(searchParams).getEntities(); } catch (AtlasBaseException e) { @@ -301,123 +305,98 @@ public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String RequestContext.get().setLexoRankCache(lexoRankCache); } - public static Map createDSLforCheckingPreExistingLexoRank(String lexoRank, String glossaryQualifiedName, String parentQualifiedName) { - - Map sortKeyOrder = mapOf(LEXICOGRAPHICAL_SORT_ORDER, mapOf("order", "desc")); - Map scoreSortOrder = mapOf("_score", mapOf("order", "desc")); - Map displayNameSortOrder = mapOf("displayName.keyword", mapOf("order", "desc")); + public static Map createDSLforCheckingPreExistingLexoRank(boolean isTerm, String lexoRank, String glossaryQualifiedName, String parentQualifiedName) { - Object[] sortArray = {sortKeyOrder, scoreSortOrder, displayNameSortOrder}; - - Map functionScore = mapOf("query", buildBoolQueryDuplicateLexoRank(lexoRank, glossaryQualifiedName, parentQualifiedName)); + Map boolMap = buildBoolQueryDuplicateLexoRank(isTerm, lexoRank, glossaryQualifiedName, parentQualifiedName); Map dsl = new HashMap<>(); dsl.put("from", 0); - dsl.put("size", 100); - dsl.put("sort", sortArray); - dsl.put("query", mapOf("function_score", functionScore)); + dsl.put("size", 1); + dsl.put("query", mapOf("bool", boolMap)); return dsl; } - private static Map buildBoolQueryDuplicateLexoRank(String lexoRank, String glossaryQualifiedName, String parentQualifiedName) { - Map boolQuery = new HashMap<>(); + private static Map buildBoolQueryDuplicateLexoRank(boolean isTerm, String lexoRank, String glossaryQualifiedName, String parentQualifiedName) { Map boolFilter = new HashMap<>(); List> mustArray = new ArrayList<>(); - List> mustNotArray = new ArrayList<>(); mustArray.add(mapOf("term", mapOf("__state", "ACTIVE"))); mustArray.add(mapOf("term", mapOf(LEXICOGRAPHICAL_SORT_ORDER, lexoRank))); if(StringUtils.isNotEmpty(glossaryQualifiedName)) { - mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossaryTerm", "AtlasGlossaryCategory")))); + mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList(ATLAS_GLOSSARY_TERM_TYPENAME, ATLAS_GLOSSARY_CATEGORY_TYPENAME)))); mustArray.add(mapOf("term", mapOf("__glossary", glossaryQualifiedName))); } else{ - mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossary")))); + mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList(ATLAS_GLOSSARY_ENTITY_TYPE)))); } if(StringUtils.isEmpty(parentQualifiedName)) { - mustNotArray.add(mapOf("exists", mapOf("field", "__categories"))); - mustNotArray.add(mapOf("exists", mapOf("field", "__parentCategory"))); + List> mustNotArray = new ArrayList<>(); + if(isTerm) { + mustNotArray.add(mapOf("exists", mapOf("field", "__categories"))); + } else { + mustNotArray.add(mapOf("exists", mapOf("field", "__parentCategory"))); + } boolFilter.put("must_not", mustNotArray); } else { List> shouldParentArray = new ArrayList<>(); - shouldParentArray.add(mapOf("term", mapOf("__categories", parentQualifiedName))); - shouldParentArray.add(mapOf("term", mapOf("__parentCategory", parentQualifiedName))); + if(isTerm) { + shouldParentArray.add(mapOf("term", mapOf("__categories", parentQualifiedName))); + } else { + shouldParentArray.add(mapOf("term", mapOf("__parentCategory", parentQualifiedName))); + } mustArray.add(mapOf("bool",mapOf("should", shouldParentArray))); } boolFilter.put("must", mustArray); - Map nestedBoolQuery = mapOf("bool", boolFilter); - - Map topBoolFilter = mapOf("filter", nestedBoolQuery); - - boolQuery.put("bool", topBoolFilter); - - return boolQuery; + return boolFilter; } - public static Map generateDSLQueryForLastCategory(String glossaryQualifiedName, String parentQualifiedName, boolean isTerm) { + public static Map generateDSLQueryForLastChild(String glossaryQualifiedName, String parentQualifiedName, boolean isTerm) { Map sortKeyOrder = mapOf(LEXICOGRAPHICAL_SORT_ORDER, mapOf("order", "desc")); - Map scoreSortOrder = mapOf("_score", mapOf("order", "desc")); - Map displayNameSortOrder = mapOf("displayName.keyword", mapOf("order", "desc")); - Object[] sortArray = {sortKeyOrder, scoreSortOrder, displayNameSortOrder}; + Object[] sortArray = {sortKeyOrder}; - Map functionScore = mapOf("query", buildBoolQuery(glossaryQualifiedName, parentQualifiedName, isTerm)); + Map boolMap = buildBoolQuery(glossaryQualifiedName, parentQualifiedName, isTerm); Map dsl = new HashMap<>(); dsl.put("from", 0); dsl.put("size", 1); dsl.put("sort", sortArray); - dsl.put("query", mapOf("function_score", functionScore)); + dsl.put("query", mapOf("bool", boolMap)); return dsl; } private static Map buildBoolQuery(String glossaryQualifiedName, String parentQualifiedName, boolean isTerm) { - Map boolQuery = new HashMap<>(); - int mustArrayLength = 0; - if(StringUtils.isEmpty(parentQualifiedName) && StringUtils.isEmpty(glossaryQualifiedName)){ - mustArrayLength = 2; - } else if(StringUtils.isEmpty(parentQualifiedName) && StringUtils.isNotEmpty(glossaryQualifiedName)){ - mustArrayLength = 3; - } else { - mustArrayLength = 4; - } - Map[] mustArray = new Map[mustArrayLength]; Map boolFilter = new HashMap<>(); - Map[] mustNotArray = new Map[2]; + List> mustArray = new ArrayList<>(); + List> mustNotArray = new ArrayList<>(); - mustArray[0] = mapOf("term", mapOf("__state", "ACTIVE")); + mustArray.add(mapOf("term", mapOf("__state", "ACTIVE"))); if(StringUtils.isNotEmpty(glossaryQualifiedName)) { String typeName = isTerm ? "AtlasGlossaryTerm" : "AtlasGlossaryCategory"; - mustArray[1] = mapOf("term", mapOf("__typeName.keyword", typeName)); - mustArray[2] = mapOf("term", mapOf("__glossary", glossaryQualifiedName)); + mustArray.add(mapOf("term", mapOf("__typeName.keyword", typeName))); + mustArray.add(mapOf("term", mapOf("__glossary", glossaryQualifiedName))); } else{ - mustArray[1] = mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossary"))); + mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossary")))); } if(StringUtils.isEmpty(parentQualifiedName)) { - mustNotArray[0] = mapOf("exists", mapOf("field", "__categories")); - mustNotArray[1] = mapOf("exists", mapOf("field", "__parentCategory")); + mustNotArray.add(mapOf("exists", mapOf("field", "__categories"))); + mustNotArray.add(mapOf("exists", mapOf("field", "__parentCategory"))); boolFilter.put("must_not", mustNotArray); } else { Map[] shouldParentArray = new Map[2]; shouldParentArray[0] = mapOf("term", mapOf("__categories", parentQualifiedName)); shouldParentArray[1] = mapOf("term", mapOf("__parentCategory", parentQualifiedName)); - mustArray[3] = mapOf("bool",mapOf("should", shouldParentArray)); + mustArray.add(mapOf("bool",mapOf("should", shouldParentArray))); } boolFilter.put("must", mustArray); - Map nestedBoolQuery = mapOf("bool", boolFilter); - - Map topBoolFilter = mapOf("filter", nestedBoolQuery); - - boolQuery.put("bool", topBoolFilter); - - return boolQuery; + return boolFilter; } } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java index baac9c82c3..c955bdd5b4 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java @@ -134,7 +134,7 @@ private void processCreateCategory(AtlasEntity entity, AtlasVertex vertex) throw if(StringUtils.isEmpty(lexicographicalSortOrder)){ assignNewLexicographicalSortOrder(entity,glossaryQualifiedName, parentQname, this.discovery); } else { - isValidLexoRank(lexicographicalSortOrder, glossaryQualifiedName, parentQname, this.discovery); + isValidLexoRank(false, lexicographicalSortOrder, glossaryQualifiedName, parentQname, this.discovery); } entity.setAttribute(QUALIFIED_NAME, createQualifiedName(vertex)); @@ -168,7 +168,7 @@ private void processUpdateCategory(AtlasEntity entity, AtlasVertex vertex) throw parentQname = (String) parentCategory.getAttribute(QUALIFIED_NAME); } if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { - isValidLexoRank(lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery); + isValidLexoRank(false, lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery); } if (!currentGlossaryQualifiedName.equals(newGlossaryQualifiedName)){ diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java index b585705284..20d14f294a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java @@ -95,7 +95,7 @@ private void processCreateGlossary(AtlasStruct entity) throws AtlasBaseException if(StringUtils.isEmpty(lexicographicalSortOrder)) { assignNewLexicographicalSortOrder((AtlasEntity) entity, null, null, this.discovery); } else { - isValidLexoRank(lexicographicalSortOrder, "", "", this.discovery); + isValidLexoRank(false, lexicographicalSortOrder, "", "", this.discovery); } entity.setAttribute(QUALIFIED_NAME, createQualifiedName()); @@ -116,7 +116,7 @@ private void processUpdateGlossary(AtlasStruct entity, AtlasVertex vertex) throw } String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { - isValidLexoRank(lexicographicalSortOrder, "", "", this.discovery); + isValidLexoRank(false, lexicographicalSortOrder, "", "", this.discovery); } String vertexQnName = vertex.getProperty(QUALIFIED_NAME, String.class); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java index 9af03cabf8..32d258b224 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java @@ -104,7 +104,7 @@ private void processCreateTerm(AtlasEntity entity, AtlasVertex vertex) throws At if(StringUtils.isEmpty(lexicographicalSortOrder)){ assignNewLexicographicalSortOrder(entity, glossaryQName, parentQname, this.discovery); } else { - isValidLexoRank(lexicographicalSortOrder, glossaryQName, parentQname, this.discovery); + isValidLexoRank(true, lexicographicalSortOrder, glossaryQName, parentQname, this.discovery); } entity.setAttribute(QUALIFIED_NAME, createQualifiedName()); @@ -137,7 +137,7 @@ private void processUpdateTerm(AtlasEntity entity, AtlasVertex vertex) throws At String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { - isValidLexoRank(lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery); + isValidLexoRank(true, lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery); } if (!currentGlossaryQualifiedName.equals(newGlossaryQualifiedName)){ diff --git a/repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem10.java b/repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem10.java deleted file mode 100644 index f2c84e1a50..0000000000 --- a/repository/src/main/java/org/apache/atlas/util/lexoRank/system/LexoNumeralSystem10.java +++ /dev/null @@ -1,32 +0,0 @@ -package org.apache.atlas.util.lexoRank.system; - -public class LexoNumeralSystem10 implements LexoNumeralSystem { - public String getName() { - return "Base10"; - } - - public int getBase() { - return 10; - } - - public char getPositiveChar() { - return '+'; - } - - public char getNegativeChar() { - return '-'; - } - - public char getRadixPointChar() { - return '.'; - } - - public int toDigit(char ch) { - if (ch >= '0' && ch <= '9') return ch - 48; - throw new IllegalArgumentException("Not valid digit: " + ch); - } - - public char toChar(int digit) { - return (char) (digit + 48); - } -} From 158748977c1aac740448b101b1daa471242cfb85 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Wed, 12 Jun 2024 22:56:34 +0530 Subject: [PATCH 12/20] Added caching for same ranks in same request, for prevention of duplicacy. --- .../graph/v2/preprocessor/PreProcessorUtils.java | 14 ++++++++++++-- .../glossary/CategoryPreProcessor.java | 4 ++-- .../glossary/GlossaryPreProcessor.java | 4 ++-- .../v2/preprocessor/glossary/TermPreProcessor.java | 4 ++-- 4 files changed, 18 insertions(+), 8 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index 98468aba76..b0bd57ab00 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -220,7 +220,7 @@ public static void verifyDuplicateAssetByName(String typeName, String assetName, } } - public static void isValidLexoRank(boolean isTerm, String input, String glossaryQualifiedName, String parentQualifiedName, EntityDiscoveryService discovery) throws AtlasBaseException { + public static void isValidLexoRank(String entityType, String input, String glossaryQualifiedName, String parentQualifiedName, EntityDiscoveryService discovery) throws AtlasBaseException { // TODO : To remove this after migration is successful on all tenants and custom-sort is successfully GA Pattern regex = Pattern.compile(LEXORANK_VALID_PATTERN); @@ -233,7 +233,15 @@ public static void isValidLexoRank(boolean isTerm, String input, String glossary if(requestFromMigration) { return; } - Map dslQuery = createDSLforCheckingPreExistingLexoRank(isTerm, input, glossaryQualifiedName, parentQualifiedName); + Map lexoRankCache = RequestContext.get().getLexoRankCache(); + if(Objects.isNull(lexoRankCache)) { + lexoRankCache = new HashMap<>(); + } + String cacheKey = entityType + "-" + glossaryQualifiedName + "-" + parentQualifiedName; + if(lexoRankCache.containsKey(cacheKey)){ + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Duplicate Lexorank found"); + } + Map dslQuery = createDSLforCheckingPreExistingLexoRank(entityType.equals(ATLAS_GLOSSARY_TERM_TYPENAME), input, glossaryQualifiedName, parentQualifiedName); List assetsWithDuplicateRank = new ArrayList<>(); try { IndexSearchParams searchParams = new IndexSearchParams(); @@ -249,6 +257,8 @@ public static void isValidLexoRank(boolean isTerm, String input, String glossary throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Duplicate Lexorank found"); } + lexoRankCache.put(cacheKey, input); + RequestContext.get().setLexoRankCache(lexoRankCache); // TODO : Add the rebalancing logic here int colonIndex = input.indexOf(":"); if (colonIndex != -1 && input.substring(colonIndex + 1).length() >= REBALANCING_TRIGGER) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java index c955bdd5b4..a321b02fec 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java @@ -134,7 +134,7 @@ private void processCreateCategory(AtlasEntity entity, AtlasVertex vertex) throw if(StringUtils.isEmpty(lexicographicalSortOrder)){ assignNewLexicographicalSortOrder(entity,glossaryQualifiedName, parentQname, this.discovery); } else { - isValidLexoRank(false, lexicographicalSortOrder, glossaryQualifiedName, parentQname, this.discovery); + isValidLexoRank(entity.getTypeName(), lexicographicalSortOrder, glossaryQualifiedName, parentQname, this.discovery); } entity.setAttribute(QUALIFIED_NAME, createQualifiedName(vertex)); @@ -168,7 +168,7 @@ private void processUpdateCategory(AtlasEntity entity, AtlasVertex vertex) throw parentQname = (String) parentCategory.getAttribute(QUALIFIED_NAME); } if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { - isValidLexoRank(false, lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery); + isValidLexoRank(entity.getTypeName(), lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery); } if (!currentGlossaryQualifiedName.equals(newGlossaryQualifiedName)){ diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java index 20d14f294a..c07322300b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java @@ -95,7 +95,7 @@ private void processCreateGlossary(AtlasStruct entity) throws AtlasBaseException if(StringUtils.isEmpty(lexicographicalSortOrder)) { assignNewLexicographicalSortOrder((AtlasEntity) entity, null, null, this.discovery); } else { - isValidLexoRank(false, lexicographicalSortOrder, "", "", this.discovery); + isValidLexoRank(entity.getTypeName(), lexicographicalSortOrder, "", "", this.discovery); } entity.setAttribute(QUALIFIED_NAME, createQualifiedName()); @@ -116,7 +116,7 @@ private void processUpdateGlossary(AtlasStruct entity, AtlasVertex vertex) throw } String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { - isValidLexoRank(false, lexicographicalSortOrder, "", "", this.discovery); + isValidLexoRank(entity.getTypeName(), lexicographicalSortOrder, "", "", this.discovery); } String vertexQnName = vertex.getProperty(QUALIFIED_NAME, String.class); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java index 32d258b224..56c08df29b 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java @@ -104,7 +104,7 @@ private void processCreateTerm(AtlasEntity entity, AtlasVertex vertex) throws At if(StringUtils.isEmpty(lexicographicalSortOrder)){ assignNewLexicographicalSortOrder(entity, glossaryQName, parentQname, this.discovery); } else { - isValidLexoRank(true, lexicographicalSortOrder, glossaryQName, parentQname, this.discovery); + isValidLexoRank(entity.getTypeName(), lexicographicalSortOrder, glossaryQName, parentQname, this.discovery); } entity.setAttribute(QUALIFIED_NAME, createQualifiedName()); @@ -137,7 +137,7 @@ private void processUpdateTerm(AtlasEntity entity, AtlasVertex vertex) throws At String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { - isValidLexoRank(true, lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery); + isValidLexoRank(entity.getTypeName(), lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery); } if (!currentGlossaryQualifiedName.equals(newGlossaryQualifiedName)){ From e28d7acb85dc0df66aaae4158d58e9fb2f9872e2 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Fri, 14 Jun 2024 11:32:38 +0530 Subject: [PATCH 13/20] Fixed a minor bug where an empty lexo attribute in update call was removing the attribute from the vertex. --- .../store/graph/v2/preprocessor/PreProcessorUtils.java | 4 ++-- .../graph/v2/preprocessor/glossary/CategoryPreProcessor.java | 3 +++ .../graph/v2/preprocessor/glossary/GlossaryPreProcessor.java | 5 ++++- .../graph/v2/preprocessor/glossary/TermPreProcessor.java | 3 +++ 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index b0bd57ab00..6df66801ae 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -226,8 +226,8 @@ public static void isValidLexoRank(String entityType, String input, String gloss Matcher matcher = regex.matcher(input); - if(!matcher.matches()){ - throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Invalid value for attribute"); + if(!matcher.matches() || StringUtils.isEmpty(input)){ + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Invalid value for lexicographicalSortOrder attribute"); } Boolean requestFromMigration = RequestContext.get().getRequestContextHeaders().getOrDefault("x-atlan-request-id", "").contains("custom-sort-migration"); if(requestFromMigration) { diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java index a321b02fec..74716d5f34 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java @@ -169,6 +169,9 @@ private void processUpdateCategory(AtlasEntity entity, AtlasVertex vertex) throw } if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { isValidLexoRank(entity.getTypeName(), lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery); + } else { + lexicographicalSortOrder = (String) storedCategory.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); + entity.setAttribute(LEXICOGRAPHICAL_SORT_ORDER, lexicographicalSortOrder); } if (!currentGlossaryQualifiedName.equals(newGlossaryQualifiedName)){ diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java index c07322300b..efdba02101 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java @@ -106,7 +106,7 @@ private void processUpdateGlossary(AtlasStruct entity, AtlasVertex vertex) throw AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdateGlossary"); String glossaryName = (String) entity.getAttribute(NAME); String vertexName = vertex.getProperty(NAME, String.class); - + AtlasEntity storedGlossary = entityRetriever.toAtlasEntity(vertex); if (!vertexName.equals(glossaryName) && glossaryExists(glossaryName)) { throw new AtlasBaseException(AtlasErrorCode.GLOSSARY_ALREADY_EXISTS,glossaryName); } @@ -117,6 +117,9 @@ private void processUpdateGlossary(AtlasStruct entity, AtlasVertex vertex) throw String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { isValidLexoRank(entity.getTypeName(), lexicographicalSortOrder, "", "", this.discovery); + } else { + lexicographicalSortOrder = (String) storedGlossary.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); + entity.setAttribute(LEXICOGRAPHICAL_SORT_ORDER, lexicographicalSortOrder); } String vertexQnName = vertex.getProperty(QUALIFIED_NAME, String.class); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java index 56c08df29b..253414e2db 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java @@ -138,6 +138,9 @@ private void processUpdateTerm(AtlasEntity entity, AtlasVertex vertex) throws At String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { isValidLexoRank(entity.getTypeName(), lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery); + } else { + lexicographicalSortOrder = (String) storedTerm.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); + entity.setAttribute(LEXICOGRAPHICAL_SORT_ORDER, lexicographicalSortOrder); } if (!currentGlossaryQualifiedName.equals(newGlossaryQualifiedName)){ From f9838dfb0ceca63c380ead08448e016b7c62ae63 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Tue, 18 Jun 2024 23:11:01 +0530 Subject: [PATCH 14/20] Fixed PR comments --- .../v2/preprocessor/PreProcessorUtils.java | 103 ++++++++---------- .../glossary/CategoryPreProcessor.java | 2 +- .../glossary/GlossaryPreProcessor.java | 3 +- .../glossary/TermPreProcessor.java | 3 +- 4 files changed, 49 insertions(+), 62 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index 6df66801ae..81ef21cfd2 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -17,6 +17,7 @@ import org.apache.atlas.util.NanoIdUtils; import org.apache.atlas.util.lexoRank.LexoRank; import org.apache.atlas.utils.AtlasEntityUtil; +import org.apache.atlas.v1.model.instance.Id; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; @@ -59,6 +60,8 @@ public class PreProcessorUtils { public static final String DATA_PRODUCT_EDGE_LABEL = "__DataDomain.dataProducts"; public static final String DOMAIN_PARENT_EDGE_LABEL = "__DataDomain.subDomains"; + public static final String STAKEHOLDER_EDGE_LABEL = "__DataDomain.stakeholders"; + public static final String PARENT_DOMAIN_QN_ATTR = "parentDomainQualifiedName"; public static final String SUPER_DOMAIN_QN_ATTR = "superDomainQualifiedName"; @@ -95,12 +98,13 @@ public enum MigrationStatus { public static final int REBALANCING_TRIGGER = 119; public static final int PRE_DELIMITER_LENGTH = 9; public static final String LEXORANK_HARD_LIMIT = "" + (256 - PRE_DELIMITER_LENGTH); - public static final String LEXORANK_VALID_PATTERN = "^0\\|[0-9a-z]{6}:(?:[0-9a-z]{0," + LEXORANK_HARD_LIMIT + "})?$"; + public static final String LEXORANK_VALID_REGEX = "^0\\|[0-9a-z]{6}:(?:[0-9a-z]{0," + LEXORANK_HARD_LIMIT + "})?$"; public static final Set ATTRIBUTES; + public static final Pattern LEXORANK_VALIDITY_PATTERN; static { - Set temp = new HashSet<>(); - temp.add(LEXICOGRAPHICAL_SORT_ORDER); - ATTRIBUTES = Collections.unmodifiableSet(temp); + ATTRIBUTES = new HashSet<>(); + ATTRIBUTES.add(LEXICOGRAPHICAL_SORT_ORDER); + LEXORANK_VALIDITY_PATTERN = Pattern.compile(LEXORANK_VALID_REGEX); } public static String getUUID(){ @@ -220,15 +224,14 @@ public static void verifyDuplicateAssetByName(String typeName, String assetName, } } - public static void isValidLexoRank(String entityType, String input, String glossaryQualifiedName, String parentQualifiedName, EntityDiscoveryService discovery) throws AtlasBaseException { - // TODO : To remove this after migration is successful on all tenants and custom-sort is successfully GA - Pattern regex = Pattern.compile(LEXORANK_VALID_PATTERN); + public static void isValidLexoRank(String entityType, String inputLexorank, String glossaryQualifiedName, String parentQualifiedName, EntityDiscoveryService discovery) throws AtlasBaseException { - Matcher matcher = regex.matcher(input); + Matcher matcher = LEXORANK_VALIDITY_PATTERN.matcher(inputLexorank); - if(!matcher.matches() || StringUtils.isEmpty(input)){ + if(!matcher.matches() || StringUtils.isEmpty(inputLexorank)){ throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Invalid value for lexicographicalSortOrder attribute"); } + // TODO : Need to discuss either to remove this after migration is successful on all tenants and custom-sort is successfully GA or keep it for re-balancing WF Boolean requestFromMigration = RequestContext.get().getRequestContextHeaders().getOrDefault("x-atlan-request-id", "").contains("custom-sort-migration"); if(requestFromMigration) { return; @@ -239,34 +242,33 @@ public static void isValidLexoRank(String entityType, String input, String gloss } String cacheKey = entityType + "-" + glossaryQualifiedName + "-" + parentQualifiedName; if(lexoRankCache.containsKey(cacheKey)){ - throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Duplicate Lexorank found"); + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Duplicate value for the attribute :" + LEXICOGRAPHICAL_SORT_ORDER +" found"); } - Map dslQuery = createDSLforCheckingPreExistingLexoRank(entityType.equals(ATLAS_GLOSSARY_TERM_TYPENAME), input, glossaryQualifiedName, parentQualifiedName); + Map dslQuery = createDSLforCheckingPreExistingLexoRank(entityType.equals(ATLAS_GLOSSARY_TERM_TYPENAME), inputLexorank, glossaryQualifiedName, parentQualifiedName); List assetsWithDuplicateRank = new ArrayList<>(); try { IndexSearchParams searchParams = new IndexSearchParams(); - searchParams.setAttributes(new HashSet<>()); searchParams.setDsl(dslQuery); assetsWithDuplicateRank = discovery.directIndexSearch(searchParams).getEntities(); } catch (AtlasBaseException e) { LOG.error("IndexSearch Error Occured : " + e.getMessage()); - new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Something went wrong with IndexSearch"); + throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Something went wrong with IndexSearch"); } if (!CollectionUtils.isEmpty(assetsWithDuplicateRank)) { throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Duplicate Lexorank found"); } - lexoRankCache.put(cacheKey, input); + lexoRankCache.put(cacheKey, inputLexorank); RequestContext.get().setLexoRankCache(lexoRankCache); // TODO : Add the rebalancing logic here - int colonIndex = input.indexOf(":"); - if (colonIndex != -1 && input.substring(colonIndex + 1).length() >= REBALANCING_TRIGGER) { +// int colonIndex = inputLexorank.indexOf(":"); +// if (colonIndex != -1 && inputLexorank.substring(colonIndex + 1).length() >= REBALANCING_TRIGGER) { // Rebalancing trigger - } +// } } - public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String glossaryQualifiedName, String parentQualifiedName, EntityDiscoveryService discovery) { + public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String glossaryQualifiedName, String parentQualifiedName, EntityDiscoveryService discovery) throws AtlasBaseException{ Map lexoRankCache = RequestContext.get().getLexoRankCache(); if(Objects.isNull(lexoRankCache)) { @@ -290,16 +292,16 @@ public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String categories = discovery.directIndexSearch(searchParams).getEntities(); } catch (AtlasBaseException e) { e.printStackTrace(); + throw new AtlasBaseException("Something went wrong in assigning lexicographicalSortOrder"); } if (CollectionUtils.isNotEmpty(categories)) { - for (AtlasEntityHeader category : categories) { - String lexicographicalSortOrder = (String) category.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); - if (StringUtils.isNotEmpty(lexicographicalSortOrder)) { - lastLexoRank = lexicographicalSortOrder; - } else { - lastLexoRank = isTerm ? INIT_TERM_LEXORANK_OFFSET : INIT_LEXORANK_OFFSET; - } + AtlasEntityHeader category = categories.get(0); + String lexicographicalSortOrder = (String) category.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); + if (StringUtils.isNotEmpty(lexicographicalSortOrder)) { + lastLexoRank = lexicographicalSortOrder; + } else { + lastLexoRank = isTerm ? INIT_TERM_LEXORANK_OFFSET : INIT_LEXORANK_OFFSET; } } else { lastLexoRank = isTerm ? INIT_TERM_LEXORANK_OFFSET : INIT_LEXORANK_OFFSET; @@ -311,7 +313,7 @@ public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String lexoRank = nextLexoRank.toString(); entity.setAttribute(LEXICOGRAPHICAL_SORT_ORDER, lexoRank); - lexoRankCache.put(entity.getTypeName() + "-" + glossaryQualifiedName + "-" + parentQualifiedName, lexoRank); + lexoRankCache.put(cacheKey, lexoRank); RequestContext.get().setLexoRankCache(lexoRankCache); } @@ -335,27 +337,14 @@ private static Map buildBoolQueryDuplicateLexoRank(boolean isTer if(StringUtils.isNotEmpty(glossaryQualifiedName)) { mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList(ATLAS_GLOSSARY_TERM_TYPENAME, ATLAS_GLOSSARY_CATEGORY_TYPENAME)))); mustArray.add(mapOf("term", mapOf("__glossary", glossaryQualifiedName))); - } else{ - mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList(ATLAS_GLOSSARY_ENTITY_TYPE)))); - } - - if(StringUtils.isEmpty(parentQualifiedName)) { - List> mustNotArray = new ArrayList<>(); - if(isTerm) { - mustNotArray.add(mapOf("exists", mapOf("field", "__categories"))); + String parentAttribute = isTerm ? "__categories" : "__parentCategory"; + if(StringUtils.isEmpty(parentQualifiedName)) { + boolFilter.put("must_not", Arrays.asList(mapOf("exists", mapOf("field", parentAttribute)))); } else { - mustNotArray.add(mapOf("exists", mapOf("field", "__parentCategory"))); + mustArray.add(mapOf("bool",mapOf("term", mapOf(parentAttribute, parentQualifiedName)))); } - boolFilter.put("must_not", mustNotArray); - } - else { - List> shouldParentArray = new ArrayList<>(); - if(isTerm) { - shouldParentArray.add(mapOf("term", mapOf("__categories", parentQualifiedName))); - } else { - shouldParentArray.add(mapOf("term", mapOf("__parentCategory", parentQualifiedName))); - } - mustArray.add(mapOf("bool",mapOf("should", shouldParentArray))); + } else{ + mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList(ATLAS_GLOSSARY_ENTITY_TYPE)))); } boolFilter.put("must", mustArray); @@ -386,25 +375,23 @@ private static Map buildBoolQuery(String glossaryQualifiedName, mustArray.add(mapOf("term", mapOf("__state", "ACTIVE"))); if(StringUtils.isNotEmpty(glossaryQualifiedName)) { - String typeName = isTerm ? "AtlasGlossaryTerm" : "AtlasGlossaryCategory"; + String typeName = isTerm ? ATLAS_GLOSSARY_TERM_TYPENAME : ATLAS_GLOSSARY_CATEGORY_TYPENAME; mustArray.add(mapOf("term", mapOf("__typeName.keyword", typeName))); mustArray.add(mapOf("term", mapOf("__glossary", glossaryQualifiedName))); + String parentAttribute = isTerm ? "__categories" : "__parentCategory"; + if(StringUtils.isEmpty(parentQualifiedName)) { + mustNotArray.add(mapOf("exists", mapOf("field", parentAttribute))); + boolFilter.put("must_not", mustNotArray); + } + else { + List> shouldParentArray = new ArrayList<>(); + shouldParentArray.add(mapOf("term", mapOf(parentAttribute, parentQualifiedName))); + mustArray.add(mapOf("bool",mapOf("should", shouldParentArray))); + } } else{ mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossary")))); } - if(StringUtils.isEmpty(parentQualifiedName)) { - mustNotArray.add(mapOf("exists", mapOf("field", "__categories"))); - mustNotArray.add(mapOf("exists", mapOf("field", "__parentCategory"))); - boolFilter.put("must_not", mustNotArray); - } - else { - Map[] shouldParentArray = new Map[2]; - shouldParentArray[0] = mapOf("term", mapOf("__categories", parentQualifiedName)); - shouldParentArray[1] = mapOf("term", mapOf("__parentCategory", parentQualifiedName)); - mustArray.add(mapOf("bool",mapOf("should", shouldParentArray))); - } - boolFilter.put("must", mustArray); return boolFilter; diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java index 74716d5f34..0c53ae5db2 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java @@ -40,7 +40,6 @@ import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasStructType; import org.apache.atlas.type.AtlasTypeRegistry; -import org.apache.atlas.util.lexoRank.LexoRank; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.collections.MapUtils; @@ -170,6 +169,7 @@ private void processUpdateCategory(AtlasEntity entity, AtlasVertex vertex) throw if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { isValidLexoRank(entity.getTypeName(), lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery); } else { + entity.removeAttribute(LEXICOGRAPHICAL_SORT_ORDER); lexicographicalSortOrder = (String) storedCategory.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); entity.setAttribute(LEXICOGRAPHICAL_SORT_ORDER, lexicographicalSortOrder); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java index efdba02101..3564cc97c9 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java @@ -106,7 +106,6 @@ private void processUpdateGlossary(AtlasStruct entity, AtlasVertex vertex) throw AtlasPerfMetrics.MetricRecorder metricRecorder = RequestContext.get().startMetricRecord("processUpdateGlossary"); String glossaryName = (String) entity.getAttribute(NAME); String vertexName = vertex.getProperty(NAME, String.class); - AtlasEntity storedGlossary = entityRetriever.toAtlasEntity(vertex); if (!vertexName.equals(glossaryName) && glossaryExists(glossaryName)) { throw new AtlasBaseException(AtlasErrorCode.GLOSSARY_ALREADY_EXISTS,glossaryName); } @@ -118,6 +117,8 @@ private void processUpdateGlossary(AtlasStruct entity, AtlasVertex vertex) throw if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { isValidLexoRank(entity.getTypeName(), lexicographicalSortOrder, "", "", this.discovery); } else { + AtlasEntity storedGlossary = entityRetriever.toAtlasEntity(vertex); + entity.removeAttribute(LEXICOGRAPHICAL_SORT_ORDER); lexicographicalSortOrder = (String) storedGlossary.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); entity.setAttribute(LEXICOGRAPHICAL_SORT_ORDER, lexicographicalSortOrder); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java index 253414e2db..6b82a9bc45 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java @@ -45,7 +45,6 @@ import org.springframework.stereotype.Component; import java.util.Iterator; import java.util.List; -import java.util.Objects; import static org.apache.atlas.repository.Constants.*; import static org.apache.atlas.repository.graph.GraphHelper.getActiveParentVertices; @@ -95,7 +94,6 @@ private void processCreateTerm(AtlasEntity entity, AtlasVertex vertex) throws At String glossaryQName = (String) anchor.getAttribute(QUALIFIED_NAME); - termExists(termName, glossaryQName); String parentQname = validateAndGetCategory(entity); @@ -139,6 +137,7 @@ private void processUpdateTerm(AtlasEntity entity, AtlasVertex vertex) throws At if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { isValidLexoRank(entity.getTypeName(), lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery); } else { + entity.removeAttribute(LEXICOGRAPHICAL_SORT_ORDER); lexicographicalSortOrder = (String) storedTerm.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); entity.setAttribute(LEXICOGRAPHICAL_SORT_ORDER, lexicographicalSortOrder); } From 586304b4101369b43329cd58e123a36978c9aa01 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Tue, 18 Jun 2024 23:15:21 +0530 Subject: [PATCH 15/20] Reverted the AbstractGlossaryPreProcessor file , previously edited. --- .../glossary/AbstractGlossaryPreProcessor.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java index 383273d73a..08c604489c 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/AbstractGlossaryPreProcessor.java @@ -26,6 +26,7 @@ import org.apache.atlas.authorize.AtlasPrivilege; import org.apache.atlas.discovery.EntityDiscoveryService; import org.apache.atlas.exception.AtlasBaseException; +import org.apache.atlas.model.discovery.IndexSearchParams; import org.apache.atlas.model.instance.AtlasEntity; import org.apache.atlas.model.instance.AtlasEntityHeader; import org.apache.atlas.model.instance.AtlasObjectId; @@ -40,14 +41,17 @@ import org.apache.atlas.tasks.TaskManagement; import org.apache.atlas.type.AtlasEntityType; import org.apache.atlas.type.AtlasTypeRegistry; -import org.apache.atlas.util.lexoRank.LexoRank; import org.apache.atlas.utils.AtlasPerfMetrics; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.util.*; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; import java.util.stream.Collectors; import static org.apache.atlas.repository.Constants.ATLAS_GLOSSARY_TERM_ENTITY_TYPE; @@ -56,8 +60,10 @@ import static org.apache.atlas.repository.Constants.STATE_PROPERTY_KEY; import static org.apache.atlas.repository.store.graph.v2.preprocessor.PreProcessorUtils.indexSearchPaginated; import static org.apache.atlas.repository.util.AtlasEntityUtils.mapOf; -import static org.apache.atlas.type.Constants.*; -import static org.apache.atlas.type.Constants.LEXICOGRAPHICAL_SORT_ORDER; +import static org.apache.atlas.type.Constants.MEANINGS_PROPERTY_KEY; +import static org.apache.atlas.type.Constants.MEANINGS_TEXT_PROPERTY_KEY; +import static org.apache.atlas.type.Constants.MEANING_NAMES_PROPERTY_KEY; +import static org.apache.atlas.type.Constants.PENDING_TASKS_PROPERTY_KEY; public abstract class AbstractGlossaryPreProcessor implements PreProcessor { private static final Logger LOG = LoggerFactory.getLogger(AbstractGlossaryPreProcessor.class); @@ -67,7 +73,6 @@ public abstract class AbstractGlossaryPreProcessor implements PreProcessor { protected static final String ATTR_MEANINGS = "meanings"; protected static final String ATTR_CATEGORIES = "categories"; - protected final AtlasTypeRegistry typeRegistry; protected final EntityGraphRetriever entityRetriever; protected final TaskManagement taskManagement; From 7ab9223419344b6dbb704acbe646e6eb418c59f7 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Thu, 20 Jun 2024 13:16:18 +0530 Subject: [PATCH 16/20] Fixed PR comments. --- .../v2/preprocessor/PreProcessorUtils.java | 23 ++++++------------- .../glossary/CategoryPreProcessor.java | 2 -- .../glossary/GlossaryPreProcessor.java | 3 --- .../glossary/TermPreProcessor.java | 2 -- 4 files changed, 7 insertions(+), 23 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index 81ef21cfd2..382ff64114 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -17,7 +17,6 @@ import org.apache.atlas.util.NanoIdUtils; import org.apache.atlas.util.lexoRank.LexoRank; import org.apache.atlas.utils.AtlasEntityUtil; -import org.apache.atlas.v1.model.instance.Id; import org.apache.commons.collections.CollectionUtils; import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; @@ -99,13 +98,9 @@ public enum MigrationStatus { public static final int PRE_DELIMITER_LENGTH = 9; public static final String LEXORANK_HARD_LIMIT = "" + (256 - PRE_DELIMITER_LENGTH); public static final String LEXORANK_VALID_REGEX = "^0\\|[0-9a-z]{6}:(?:[0-9a-z]{0," + LEXORANK_HARD_LIMIT + "})?$"; - public static final Set ATTRIBUTES; - public static final Pattern LEXORANK_VALIDITY_PATTERN; - static { - ATTRIBUTES = new HashSet<>(); - ATTRIBUTES.add(LEXICOGRAPHICAL_SORT_ORDER); - LEXORANK_VALIDITY_PATTERN = Pattern.compile(LEXORANK_VALID_REGEX); - } + public static final Set ATTRIBUTES = new HashSet<>(Arrays.asList("LEXICOGRAPHICAL_SORT_ORDER")); + + public static final Pattern LEXORANK_VALIDITY_PATTERN = Pattern.compile(LEXORANK_VALID_REGEX); public static String getUUID(){ return NanoIdUtils.randomNanoId(); @@ -341,7 +336,7 @@ private static Map buildBoolQueryDuplicateLexoRank(boolean isTer if(StringUtils.isEmpty(parentQualifiedName)) { boolFilter.put("must_not", Arrays.asList(mapOf("exists", mapOf("field", parentAttribute)))); } else { - mustArray.add(mapOf("bool",mapOf("term", mapOf(parentAttribute, parentQualifiedName)))); + mustArray.add(mapOf("term", mapOf(parentAttribute, parentQualifiedName))); } } else{ mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList(ATLAS_GLOSSARY_ENTITY_TYPE)))); @@ -351,6 +346,7 @@ private static Map buildBoolQueryDuplicateLexoRank(boolean isTer return boolFilter; } + public static Map generateDSLQueryForLastChild(String glossaryQualifiedName, String parentQualifiedName, boolean isTerm) { Map sortKeyOrder = mapOf(LEXICOGRAPHICAL_SORT_ORDER, mapOf("order", "desc")); @@ -371,8 +367,6 @@ public static Map generateDSLQueryForLastChild(String glossaryQu private static Map buildBoolQuery(String glossaryQualifiedName, String parentQualifiedName, boolean isTerm) { Map boolFilter = new HashMap<>(); List> mustArray = new ArrayList<>(); - List> mustNotArray = new ArrayList<>(); - mustArray.add(mapOf("term", mapOf("__state", "ACTIVE"))); if(StringUtils.isNotEmpty(glossaryQualifiedName)) { String typeName = isTerm ? ATLAS_GLOSSARY_TERM_TYPENAME : ATLAS_GLOSSARY_CATEGORY_TYPENAME; @@ -380,13 +374,10 @@ private static Map buildBoolQuery(String glossaryQualifiedName, mustArray.add(mapOf("term", mapOf("__glossary", glossaryQualifiedName))); String parentAttribute = isTerm ? "__categories" : "__parentCategory"; if(StringUtils.isEmpty(parentQualifiedName)) { - mustNotArray.add(mapOf("exists", mapOf("field", parentAttribute))); - boolFilter.put("must_not", mustNotArray); + boolFilter.put("must_not", Arrays.asList(mapOf("exists", mapOf("field", parentAttribute)))); } else { - List> shouldParentArray = new ArrayList<>(); - shouldParentArray.add(mapOf("term", mapOf(parentAttribute, parentQualifiedName))); - mustArray.add(mapOf("bool",mapOf("should", shouldParentArray))); + mustArray.add(mapOf("term", mapOf(parentAttribute, parentQualifiedName))); } } else{ mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossary")))); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java index 0c53ae5db2..fbc51477e2 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java @@ -170,8 +170,6 @@ private void processUpdateCategory(AtlasEntity entity, AtlasVertex vertex) throw isValidLexoRank(entity.getTypeName(), lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery); } else { entity.removeAttribute(LEXICOGRAPHICAL_SORT_ORDER); - lexicographicalSortOrder = (String) storedCategory.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); - entity.setAttribute(LEXICOGRAPHICAL_SORT_ORDER, lexicographicalSortOrder); } if (!currentGlossaryQualifiedName.equals(newGlossaryQualifiedName)){ diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java index 3564cc97c9..c8023e3611 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java @@ -117,10 +117,7 @@ private void processUpdateGlossary(AtlasStruct entity, AtlasVertex vertex) throw if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { isValidLexoRank(entity.getTypeName(), lexicographicalSortOrder, "", "", this.discovery); } else { - AtlasEntity storedGlossary = entityRetriever.toAtlasEntity(vertex); entity.removeAttribute(LEXICOGRAPHICAL_SORT_ORDER); - lexicographicalSortOrder = (String) storedGlossary.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); - entity.setAttribute(LEXICOGRAPHICAL_SORT_ORDER, lexicographicalSortOrder); } String vertexQnName = vertex.getProperty(QUALIFIED_NAME, String.class); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java index 6b82a9bc45..761c8673f2 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java @@ -138,8 +138,6 @@ private void processUpdateTerm(AtlasEntity entity, AtlasVertex vertex) throws At isValidLexoRank(entity.getTypeName(), lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery); } else { entity.removeAttribute(LEXICOGRAPHICAL_SORT_ORDER); - lexicographicalSortOrder = (String) storedTerm.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); - entity.setAttribute(LEXICOGRAPHICAL_SORT_ORDER, lexicographicalSortOrder); } if (!currentGlossaryQualifiedName.equals(newGlossaryQualifiedName)){ From cb56a9408d3a8c84cd8e742b1e92a8139d149223 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Tue, 25 Jun 2024 20:02:10 +0530 Subject: [PATCH 17/20] fixed attribute name in dsl query --- .../store/graph/v2/preprocessor/PreProcessorUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index 382ff64114..187c8752a6 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -98,7 +98,7 @@ public enum MigrationStatus { public static final int PRE_DELIMITER_LENGTH = 9; public static final String LEXORANK_HARD_LIMIT = "" + (256 - PRE_DELIMITER_LENGTH); public static final String LEXORANK_VALID_REGEX = "^0\\|[0-9a-z]{6}:(?:[0-9a-z]{0," + LEXORANK_HARD_LIMIT + "})?$"; - public static final Set ATTRIBUTES = new HashSet<>(Arrays.asList("LEXICOGRAPHICAL_SORT_ORDER")); + public static final Set ATTRIBUTES = new HashSet<>(Arrays.asList("lexicographicalSortOrder")); public static final Pattern LEXORANK_VALIDITY_PATTERN = Pattern.compile(LEXORANK_VALID_REGEX); From 1bd6f9b7cfe0bdf152b0cd6ccb0ce8477e7dc54d Mon Sep 17 00:00:00 2001 From: hr2904 Date: Fri, 28 Jun 2024 14:42:00 +0530 Subject: [PATCH 18/20] Fixed a caching logic while checking for duplicate ranks. --- .../store/graph/v2/preprocessor/PreProcessorUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index 187c8752a6..70bee139c3 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -236,7 +236,7 @@ public static void isValidLexoRank(String entityType, String inputLexorank, Stri lexoRankCache = new HashMap<>(); } String cacheKey = entityType + "-" + glossaryQualifiedName + "-" + parentQualifiedName; - if(lexoRankCache.containsKey(cacheKey)){ + if(lexoRankCache.containsKey(cacheKey) && lexoRankCache.get(cacheKey).equals(inputLexorank)){ throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Duplicate value for the attribute :" + LEXICOGRAPHICAL_SORT_ORDER +" found"); } Map dslQuery = createDSLforCheckingPreExistingLexoRank(entityType.equals(ATLAS_GLOSSARY_TERM_TYPENAME), inputLexorank, glossaryQualifiedName, parentQualifiedName); From 0d139eadd8a18e979ea3402db60e5b5d1844fc22 Mon Sep 17 00:00:00 2001 From: hr2904 Date: Fri, 5 Jul 2024 18:14:04 +0530 Subject: [PATCH 19/20] Removed term-category bifurcation as we are now allowing term-category intersperse. --- .../v2/preprocessor/PreProcessorUtils.java | 49 ++++++++++--------- .../glossary/CategoryPreProcessor.java | 4 +- .../glossary/GlossaryPreProcessor.java | 4 +- .../glossary/TermPreProcessor.java | 4 +- 4 files changed, 31 insertions(+), 30 deletions(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index 70bee139c3..20d0febe1a 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -219,7 +219,7 @@ public static void verifyDuplicateAssetByName(String typeName, String assetName, } } - public static void isValidLexoRank(String entityType, String inputLexorank, String glossaryQualifiedName, String parentQualifiedName, EntityDiscoveryService discovery) throws AtlasBaseException { + public static void isValidLexoRank(String inputLexorank, String glossaryQualifiedName, String parentQualifiedName, EntityDiscoveryService discovery) throws AtlasBaseException { Matcher matcher = LEXORANK_VALIDITY_PATTERN.matcher(inputLexorank); @@ -235,11 +235,11 @@ public static void isValidLexoRank(String entityType, String inputLexorank, Stri if(Objects.isNull(lexoRankCache)) { lexoRankCache = new HashMap<>(); } - String cacheKey = entityType + "-" + glossaryQualifiedName + "-" + parentQualifiedName; + String cacheKey = glossaryQualifiedName + "-" + parentQualifiedName; if(lexoRankCache.containsKey(cacheKey) && lexoRankCache.get(cacheKey).equals(inputLexorank)){ throw new AtlasBaseException(AtlasErrorCode.BAD_REQUEST, "Duplicate value for the attribute :" + LEXICOGRAPHICAL_SORT_ORDER +" found"); } - Map dslQuery = createDSLforCheckingPreExistingLexoRank(entityType.equals(ATLAS_GLOSSARY_TERM_TYPENAME), inputLexorank, glossaryQualifiedName, parentQualifiedName); + Map dslQuery = createDSLforCheckingPreExistingLexoRank(inputLexorank, glossaryQualifiedName, parentQualifiedName); List assetsWithDuplicateRank = new ArrayList<>(); try { IndexSearchParams searchParams = new IndexSearchParams(); @@ -271,15 +271,14 @@ public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String } String lexoRank = ""; String lastLexoRank = ""; - boolean isTerm = entity.getTypeName().equals(ATLAS_GLOSSARY_TERM_TYPENAME); - String cacheKey = entity.getTypeName() + "-" + glossaryQualifiedName + "-" + parentQualifiedName; + String cacheKey = glossaryQualifiedName + "-" + parentQualifiedName; if(lexoRankCache.containsKey(cacheKey)) { lastLexoRank = lexoRankCache.get(cacheKey); } else { List categories = null; - Map dslQuery = generateDSLQueryForLastChild(glossaryQualifiedName, parentQualifiedName, isTerm); + Map dslQuery = generateDSLQueryForLastChild(glossaryQualifiedName, parentQualifiedName); try { IndexSearchParams searchParams = new IndexSearchParams(); searchParams.setAttributes(ATTRIBUTES); @@ -296,10 +295,10 @@ public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String if (StringUtils.isNotEmpty(lexicographicalSortOrder)) { lastLexoRank = lexicographicalSortOrder; } else { - lastLexoRank = isTerm ? INIT_TERM_LEXORANK_OFFSET : INIT_LEXORANK_OFFSET; + lastLexoRank = INIT_LEXORANK_OFFSET; } } else { - lastLexoRank = isTerm ? INIT_TERM_LEXORANK_OFFSET : INIT_LEXORANK_OFFSET; + lastLexoRank = INIT_LEXORANK_OFFSET; } } @@ -312,9 +311,9 @@ public static void assignNewLexicographicalSortOrder(AtlasEntity entity, String RequestContext.get().setLexoRankCache(lexoRankCache); } - public static Map createDSLforCheckingPreExistingLexoRank(boolean isTerm, String lexoRank, String glossaryQualifiedName, String parentQualifiedName) { + public static Map createDSLforCheckingPreExistingLexoRank(String lexoRank, String glossaryQualifiedName, String parentQualifiedName) { - Map boolMap = buildBoolQueryDuplicateLexoRank(isTerm, lexoRank, glossaryQualifiedName, parentQualifiedName); + Map boolMap = buildBoolQueryDuplicateLexoRank(lexoRank, glossaryQualifiedName, parentQualifiedName); Map dsl = new HashMap<>(); dsl.put("from", 0); @@ -324,7 +323,7 @@ public static Map createDSLforCheckingPreExistingLexoRank(boolea return dsl; } - private static Map buildBoolQueryDuplicateLexoRank(boolean isTerm, String lexoRank, String glossaryQualifiedName, String parentQualifiedName) { + private static Map buildBoolQueryDuplicateLexoRank(String lexoRank, String glossaryQualifiedName, String parentQualifiedName) { Map boolFilter = new HashMap<>(); List> mustArray = new ArrayList<>(); mustArray.add(mapOf("term", mapOf("__state", "ACTIVE"))); @@ -332,11 +331,13 @@ private static Map buildBoolQueryDuplicateLexoRank(boolean isTer if(StringUtils.isNotEmpty(glossaryQualifiedName)) { mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList(ATLAS_GLOSSARY_TERM_TYPENAME, ATLAS_GLOSSARY_CATEGORY_TYPENAME)))); mustArray.add(mapOf("term", mapOf("__glossary", glossaryQualifiedName))); - String parentAttribute = isTerm ? "__categories" : "__parentCategory"; if(StringUtils.isEmpty(parentQualifiedName)) { - boolFilter.put("must_not", Arrays.asList(mapOf("exists", mapOf("field", parentAttribute)))); + boolFilter.put("must_not", Arrays.asList(mapOf("exists", mapOf("field", "__categories")),mapOf("exists", mapOf("field", "__parentCategory")))); } else { - mustArray.add(mapOf("term", mapOf(parentAttribute, parentQualifiedName))); + List> shouldParentArray = new ArrayList<>(); + shouldParentArray.add(mapOf("term", mapOf("__categories", parentQualifiedName))); + shouldParentArray.add(mapOf("term", mapOf("__parentCategory", parentQualifiedName))); + mustArray.add(mapOf("bool",mapOf("should", shouldParentArray))); } } else{ mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList(ATLAS_GLOSSARY_ENTITY_TYPE)))); @@ -347,13 +348,13 @@ private static Map buildBoolQueryDuplicateLexoRank(boolean isTer return boolFilter; } - public static Map generateDSLQueryForLastChild(String glossaryQualifiedName, String parentQualifiedName, boolean isTerm) { + public static Map generateDSLQueryForLastChild(String glossaryQualifiedName, String parentQualifiedName) { Map sortKeyOrder = mapOf(LEXICOGRAPHICAL_SORT_ORDER, mapOf("order", "desc")); Object[] sortArray = {sortKeyOrder}; - Map boolMap = buildBoolQuery(glossaryQualifiedName, parentQualifiedName, isTerm); + Map boolMap = buildBoolQuery(glossaryQualifiedName, parentQualifiedName); Map dsl = new HashMap<>(); dsl.put("from", 0); @@ -364,20 +365,20 @@ public static Map generateDSLQueryForLastChild(String glossaryQu return dsl; } - private static Map buildBoolQuery(String glossaryQualifiedName, String parentQualifiedName, boolean isTerm) { + private static Map buildBoolQuery(String glossaryQualifiedName, String parentQualifiedName) { Map boolFilter = new HashMap<>(); List> mustArray = new ArrayList<>(); mustArray.add(mapOf("term", mapOf("__state", "ACTIVE"))); if(StringUtils.isNotEmpty(glossaryQualifiedName)) { - String typeName = isTerm ? ATLAS_GLOSSARY_TERM_TYPENAME : ATLAS_GLOSSARY_CATEGORY_TYPENAME; - mustArray.add(mapOf("term", mapOf("__typeName.keyword", typeName))); + mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossaryTerm", "AtlasGlossaryCategory")))); mustArray.add(mapOf("term", mapOf("__glossary", glossaryQualifiedName))); - String parentAttribute = isTerm ? "__categories" : "__parentCategory"; if(StringUtils.isEmpty(parentQualifiedName)) { - boolFilter.put("must_not", Arrays.asList(mapOf("exists", mapOf("field", parentAttribute)))); - } - else { - mustArray.add(mapOf("term", mapOf(parentAttribute, parentQualifiedName))); + boolFilter.put("must_not", Arrays.asList(mapOf("exists", mapOf("field", "__categories")),mapOf("exists", mapOf("field", "__parentCategory")))); + } else { + List> shouldParentArray = new ArrayList<>(); + shouldParentArray.add(mapOf("term", mapOf("__categories", parentQualifiedName))); + shouldParentArray.add(mapOf("term", mapOf("__parentCategory", parentQualifiedName))); + mustArray.add(mapOf("bool",mapOf("should", shouldParentArray))); } } else{ mustArray.add(mapOf("terms", mapOf("__typeName.keyword", Arrays.asList("AtlasGlossary")))); diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java index fbc51477e2..4f333d5468 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/CategoryPreProcessor.java @@ -133,7 +133,7 @@ private void processCreateCategory(AtlasEntity entity, AtlasVertex vertex) throw if(StringUtils.isEmpty(lexicographicalSortOrder)){ assignNewLexicographicalSortOrder(entity,glossaryQualifiedName, parentQname, this.discovery); } else { - isValidLexoRank(entity.getTypeName(), lexicographicalSortOrder, glossaryQualifiedName, parentQname, this.discovery); + isValidLexoRank(lexicographicalSortOrder, glossaryQualifiedName, parentQname, this.discovery); } entity.setAttribute(QUALIFIED_NAME, createQualifiedName(vertex)); @@ -167,7 +167,7 @@ private void processUpdateCategory(AtlasEntity entity, AtlasVertex vertex) throw parentQname = (String) parentCategory.getAttribute(QUALIFIED_NAME); } if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { - isValidLexoRank(entity.getTypeName(), lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery); + isValidLexoRank(lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery); } else { entity.removeAttribute(LEXICOGRAPHICAL_SORT_ORDER); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java index c8023e3611..fc0bec0654 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/GlossaryPreProcessor.java @@ -95,7 +95,7 @@ private void processCreateGlossary(AtlasStruct entity) throws AtlasBaseException if(StringUtils.isEmpty(lexicographicalSortOrder)) { assignNewLexicographicalSortOrder((AtlasEntity) entity, null, null, this.discovery); } else { - isValidLexoRank(entity.getTypeName(), lexicographicalSortOrder, "", "", this.discovery); + isValidLexoRank(lexicographicalSortOrder, "", "", this.discovery); } entity.setAttribute(QUALIFIED_NAME, createQualifiedName()); @@ -115,7 +115,7 @@ private void processUpdateGlossary(AtlasStruct entity, AtlasVertex vertex) throw } String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { - isValidLexoRank(entity.getTypeName(), lexicographicalSortOrder, "", "", this.discovery); + isValidLexoRank(lexicographicalSortOrder, "", "", this.discovery); } else { entity.removeAttribute(LEXICOGRAPHICAL_SORT_ORDER); } diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java index 761c8673f2..e6195787fa 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/glossary/TermPreProcessor.java @@ -102,7 +102,7 @@ private void processCreateTerm(AtlasEntity entity, AtlasVertex vertex) throws At if(StringUtils.isEmpty(lexicographicalSortOrder)){ assignNewLexicographicalSortOrder(entity, glossaryQName, parentQname, this.discovery); } else { - isValidLexoRank(entity.getTypeName(), lexicographicalSortOrder, glossaryQName, parentQname, this.discovery); + isValidLexoRank(lexicographicalSortOrder, glossaryQName, parentQname, this.discovery); } entity.setAttribute(QUALIFIED_NAME, createQualifiedName()); @@ -135,7 +135,7 @@ private void processUpdateTerm(AtlasEntity entity, AtlasVertex vertex) throws At String lexicographicalSortOrder = (String) entity.getAttribute(LEXICOGRAPHICAL_SORT_ORDER); if(StringUtils.isNotEmpty(lexicographicalSortOrder)) { - isValidLexoRank(entity.getTypeName(), lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery); + isValidLexoRank(lexicographicalSortOrder, newGlossaryQualifiedName, parentQname, this.discovery); } else { entity.removeAttribute(LEXICOGRAPHICAL_SORT_ORDER); } From 075f5ae0397156b7a4e27d70c6428715eeb9f38e Mon Sep 17 00:00:00 2001 From: hr2904 Date: Wed, 9 Oct 2024 17:18:18 +0530 Subject: [PATCH 20/20] Removed an unneeded constant --- .../store/graph/v2/preprocessor/PreProcessorUtils.java | 1 - 1 file changed, 1 deletion(-) diff --git a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java index 20d0febe1a..e5171fde92 100644 --- a/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java +++ b/repository/src/main/java/org/apache/atlas/repository/store/graph/v2/preprocessor/PreProcessorUtils.java @@ -45,7 +45,6 @@ public class PreProcessorUtils { public static final String GLOSSARY_TERM_REL_TYPE = "AtlasGlossaryTermAnchor"; public static final String GLOSSARY_CATEGORY_REL_TYPE = "AtlasGlossaryCategoryAnchor"; public static final String INIT_LEXORANK_OFFSET = "0|100000:"; - public static final String INIT_TERM_LEXORANK_OFFSET = "0|500000:"; //DataMesh models constants public static final String PARENT_DOMAIN_REL_TYPE = "parentDomain";