diff --git a/here-naksha-lib-base/src/commonMain/kotlin/naksha/base/NormalizerForm.kt b/here-naksha-lib-base/src/commonMain/kotlin/naksha/base/NormalizerForm.kt index 04ded0e6b..852ba575a 100644 --- a/here-naksha-lib-base/src/commonMain/kotlin/naksha/base/NormalizerForm.kt +++ b/here-naksha-lib-base/src/commonMain/kotlin/naksha/base/NormalizerForm.kt @@ -4,7 +4,7 @@ import kotlin.js.ExperimentalJsExport import kotlin.js.JsExport /** - * Please refer []Unicode Normalization Forms](https://www.unicode.org/reports/tr15/#Norm_Forms) + * Please refer [Unicode Normalization Forms](https://www.unicode.org/reports/tr15/#Norm_Forms) */ @OptIn(ExperimentalJsExport::class) @JsExport diff --git a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/Tag.kt b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/Tag.kt index 1537e6b78..4174e7121 100644 --- a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/Tag.kt +++ b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/Tag.kt @@ -3,6 +3,8 @@ package naksha.model import naksha.base.* +import naksha.base.NormalizerForm.NFD +import naksha.base.NormalizerForm.NFKC import naksha.model.NakshaError.NakshaErrorCompanion.ILLEGAL_ARGUMENT import kotlin.js.JsExport import kotlin.js.JsName @@ -16,10 +18,10 @@ import kotlin.jvm.JvmStatic * @property value the value of the tag; _null_, Boolean, String or Double. */ @JsExport -class Tag(): AnyObject() { +class Tag() : AnyObject() { @JsName("of") - constructor(tag: String, key: String, value: Any?): this() { + constructor(tag: String, key: String, value: Any?) : this() { this.tag = tag this.key = key this.value = value @@ -32,42 +34,19 @@ class Tag(): AnyObject() { @JvmStatic @JsStatic - fun parse(tag: String): Tag { - val i = tag.indexOf('=') - val key: String - val value: Any? - if (i > 1) { - if (tag[i-1] == ':') { // := - key = tag.substring(0, i-1).trim() - val raw = tag.substring(i + 1).trim() - value = if ("true".equals(raw, ignoreCase = true)) { - true - } else if ("false".equals(raw, ignoreCase = true)) { - false - } else { - raw. toDouble() - } - } else { - key = tag.substring(0, i).trim() - value = tag.substring(i + 1).trim() - } - } else { - key = tag - value = null + fun of(normalizedKey: String, normalizedValue: Any?): Tag = when (normalizedValue) { + null -> Tag(normalizedKey, normalizedKey, null) + is String -> Tag("$normalizedKey=$normalizedValue", normalizedKey, normalizedValue) + is Boolean -> Tag("$normalizedKey:=$normalizedValue", normalizedKey, normalizedValue) + is Number -> { + val doubleValue = normalizedValue.toDouble() + Tag("$normalizedKey:=$doubleValue", normalizedKey, doubleValue) } - return Tag(tag, key, value) - } - @JvmStatic - @JsStatic - fun of(key: String, value: Any?): Tag = when(value) { - // TODO: Fix normalization! - null -> Tag(key, key, null) - is String -> Tag("$key=$value", key, value) - is Boolean, Double -> Tag("$key:=$value", key, value) - is Number -> of(key, value.toDouble()) - is Int64 -> of(key, value.toDouble()) - else -> throw NakshaException(ILLEGAL_ARGUMENT, "Tag values can only be String, Boolean or Double") + else -> throw NakshaException( + ILLEGAL_ARGUMENT, + "Tag values can only be String, Boolean or Number" + ) } } @@ -81,6 +60,7 @@ class Tag(): AnyObject() { if (other is Tag) return tag == other.tag return false } + override fun hashCode(): Int = tag.hashCode() override fun toString(): String = tag } \ No newline at end of file diff --git a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagList.kt b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagList.kt index f77d0e299..e53c55143 100644 --- a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagList.kt +++ b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagList.kt @@ -3,7 +3,9 @@ package naksha.model import naksha.base.ListProxy -import naksha.model.XyzNs.XyzNsCompanion.normalizeTag +import naksha.base.NormalizerForm +import naksha.base.Platform +import naksha.model.TagNormalizer.normalizeTag import kotlin.js.JsExport import kotlin.js.JsName import kotlin.js.JsStatic @@ -24,21 +26,6 @@ open class TagList() : ListProxy(String::class) { addTags(listOf(*tags), false) } - companion object TagList_C { - /** - * Create a tag list from the given array; the tags are normalized. - * @param tags the tags. - * @return the tag-list. - */ - @JvmStatic - @JsStatic - fun fromArray(tags: Array): TagList { - val list = TagList() - list.addAndNormalizeTags(*tags) - return list - } - } - /** * Returns 'true' if the tag was removed, 'false' if it was not present. * @@ -161,9 +148,44 @@ open class TagList() : ListProxy(String::class) { return this } + /** * Convert this tag-list into a tag-map. * @return this tag-list as tag-map. */ fun toTagMap(): TagMap = TagMap(this) + + companion object TagList_C { + /** + * Create a tag list from the given array; the tags are normalized. + * @param tags the tags. + * @return the tag-list. + */ + @JvmStatic + @JsStatic + fun fromArray(tags: Array): TagList { + val list = TagList() + list.addAndNormalizeTags(*tags) + return list + } + + /** + * A method to normalize a list of tags. + * + * @param tags a list of tags. + * @return the same list, just that the content is normalized. + */ + @JvmStatic + @JsStatic + fun normalizeTags(tags: TagList?): TagList? { + if (!tags.isNullOrEmpty()) { + for ((idx, tag) in tags.withIndex()) { + if (tag != null) { + tags[idx] = normalizeTag(tag) + } + } + } + return tags + } + } } diff --git a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagMap.kt b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagMap.kt index dbb734a05..eed811048 100644 --- a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagMap.kt +++ b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagMap.kt @@ -3,7 +3,6 @@ package naksha.model import naksha.base.MapProxy -import naksha.model.request.query.* import kotlin.js.JsExport import kotlin.js.JsName @@ -11,16 +10,14 @@ import kotlin.js.JsName // Improve me! @JsExport -open class TagMap() : MapProxy(String::class, Tag::class) { +open class TagMap() : MapProxy(String::class, Any::class) { @Suppress("LeakingThis") @JsName("of") - constructor(tagList: TagList) : this(){ - for (s in tagList) { - if (s == null) continue - val tag = Tag.parse(s) - put(tag.key, tag) - } + constructor(tagList: TagList) : this() { + tagList.filterNotNull() + .map { TagNormalizer.splitNormalizedTag(it) } + .forEach { tag -> put(tag.key, tag.value) } } /** @@ -29,9 +26,8 @@ open class TagMap() : MapProxy(String::class, Tag::class) { */ fun toTagList(): TagList { val list = TagList() - for (e in this) { - val tag = e.value?.tag - if (tag != null) list.add(tag) + forEach { (key, value) -> + list.add(Tag.of(key, value).tag) } return list } diff --git a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagNormalizer.kt b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagNormalizer.kt new file mode 100644 index 000000000..90e217767 --- /dev/null +++ b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/TagNormalizer.kt @@ -0,0 +1,110 @@ +package naksha.model + +import naksha.base.NormalizerForm +import naksha.base.NormalizerForm.NFD +import naksha.base.NormalizerForm.NFKC +import naksha.base.Platform +import naksha.model.TagNormalizer.normalizeTag +import naksha.model.TagNormalizer.splitNormalizedTag + +/** + * An object used for Tag normalization and splitting. + * + * Process of normalization happens in [normalizeTag] method and includes following steps: + * 1) Always: apply normalization form (see [NormalizerForm]) + * 2) Conditional: lowercase the whole tag + * 3) Conditional: remove all non-ASCII characters + * + * Normalization form used in step #1 and subsequent conditional steps depend on tag prefix. + * + * + * Process of splitting happens in [splitNormalizedTag] method. + * It is about splitting the normalized tag to [key, value] pair in form of [Tag] + * Note that not all tags can be split, it depends on their prefix. + * + * Summarised per-prefix behavior: + * +----------+------------+-----------+----------+-------+ + * | prefix | norm. form | lowercase | no ASCII | split | + * +----------+------------+-----------+----------+-------+ + * | @ | NFKC | false | false | true | + * | ref_ | NFKC | false | false | false | + * | ~ | NFD | false | true | true | + * | # | NFD | false | true | true | + * | sourceID | NFKC | false | false | false | + * | < ELSE > | NFD | true | true | true | + * +----------+------------+-----------+----------+-------+ + * + * By default, (if no special prefix is found) tag is normalized with NFD, lowercased, cleaned of non-ASCII and splittable. + */ +object TagNormalizer { + private data class TagProcessingPolicy( + val normalizerForm: NormalizerForm, + val removeNonAscii: Boolean, + val lowercase: Boolean, + val split: Boolean + ) + + private val DEFAULT_POLICY = TagProcessingPolicy(NFD, removeNonAscii = true, lowercase = true, split = true) + private val PREFIX_TO_POLICY = mapOf( + "@" to TagProcessingPolicy(NFKC, removeNonAscii = false, lowercase = false, split = true), + "ref_" to TagProcessingPolicy(NFKC, removeNonAscii = false, lowercase = false, split = false), + "sourceID" to TagProcessingPolicy(NFKC, removeNonAscii = false, lowercase = false, split = false), + "~" to TagProcessingPolicy(NFD, removeNonAscii = true, lowercase = false, split = true), + "#" to TagProcessingPolicy(NFD, removeNonAscii = true, lowercase = false, split = true) + ) + + private val PRINTABLE_ASCII_CODES = 32..128 + + /** + * Main method for raw tag normalization. See[TagNormalizer] doc for more + */ + fun normalizeTag(tag: String): String { + val policy = policyFor(tag) + var normalized = Platform.normalize(tag, policy.normalizerForm) + normalized = if (policy.lowercase) normalized.lowercase() else normalized + normalized = if (policy.removeNonAscii) removeNonAscii(normalized) else normalized + return normalized + } + + /** + * Main method for normalized tag splitting. See[TagNormalizer] doc for more + */ + fun splitNormalizedTag(normalizedTag: String): Tag { + if (!policyFor(normalizedTag).split) { + return Tag.of(normalizedKey = normalizedTag, normalizedValue = null) + } + val i = normalizedTag.indexOf('=') + val key: String + val value: Any? + if (i > 1) { + if (normalizedTag[i - 1] == ':') { // := + key = normalizedTag.substring(0, i - 1).trim() + val raw = normalizedTag.substring(i + 1).trim() + value = if ("true".equals(raw, ignoreCase = true)) { + true + } else if ("false".equals(raw, ignoreCase = true)) { + false + } else { + raw.toDouble() + } + } else { + key = normalizedTag.substring(0, i).trim() + value = normalizedTag.substring(i + 1).trim() + } + } else { + key = normalizedTag + value = null + } + return Tag(normalizedTag, key, value) + } + + private fun removeNonAscii(text: String) = + text.filter { it.code in PRINTABLE_ASCII_CODES } + + private fun policyFor(tag: String): TagProcessingPolicy { + return PREFIX_TO_POLICY.entries + .firstOrNull { (prefix, _) -> tag.startsWith(prefix, ignoreCase = true) } + ?.value + ?: DEFAULT_POLICY + } +} \ No newline at end of file diff --git a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/XyzNs.kt b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/XyzNs.kt index 1a3a9b256..acd8c14b7 100644 --- a/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/XyzNs.kt +++ b/here-naksha-lib-model/src/commonMain/kotlin/naksha/model/XyzNs.kt @@ -3,6 +3,7 @@ package naksha.model import naksha.base.* +import naksha.model.TagNormalizer.normalizeTag import kotlin.DeprecationLevel.WARNING import kotlin.js.JsExport import kotlin.js.JsStatic @@ -29,66 +30,7 @@ class XyzNs : AnyObject() { private val INT64_NULL = NullableProperty(Int64::class) private val TAGS = NullableProperty(TagList::class) - private var AS_IS: CharArray = CharArray(128 - 32) { (it + 32).toChar() } - private var TO_LOWER: CharArray = CharArray(128 - 32) { (it + 32).toChar().lowercaseChar() } - - /** - * A method to normalize a list of tags. - * - * @param tags a list of tags. - * @return the same list, just that the content is normalized. - */ - @JvmStatic - @JsStatic - fun normalizeTags(tags: TagList?): TagList? { - if (!tags.isNullOrEmpty()) { - for ((idx, tag) in tags.withIndex()) { - if (tag != null) { - tags[idx] = normalizeTag(tag) - } - } - } - return tags - } - /** - * A method to normalize and lower case a tag. - * - * @param tag the tag. - * @return the normalized and lower cased version of it. - */ - @JvmStatic - @JsStatic - fun normalizeTag(tag: String): String { - if (tag.isEmpty()) { - return tag - } - val first = tag[0] - // All tags starting with an at-sign, will not be modified in any way. - if (first == '@') { - return tag - } - - // Normalize the tag. - val normalized: String = Platform.normalize(tag, NormalizerForm.NFD) - - // All tags starting with a tilde, sharp, or the deprecated "ref_" / "sourceID_" prefix will not - // be lower cased. - val MAP: CharArray = - if (first == '~' || first == '#' || normalized.startsWith("ref_") || normalized.startsWith("sourceID_")) - AS_IS - else - TO_LOWER - val sb = StringBuilder(normalized.length) - for (element in normalized) { - // Note: This saves one branch, and the array-size check, because 0 - 32 will become 65504. - val c = (element.code - 32).toChar() - if (c.code < MAP.size) { - sb.append(MAP[c.code]) - } - } - return sb.toString() - } } /** diff --git a/here-naksha-lib-psql/src/commonMain/kotlin/naksha/psql/executors/query/WhereClauseBuilder.kt b/here-naksha-lib-psql/src/commonMain/kotlin/naksha/psql/executors/query/WhereClauseBuilder.kt index 760bf133b..55a346d1a 100644 --- a/here-naksha-lib-psql/src/commonMain/kotlin/naksha/psql/executors/query/WhereClauseBuilder.kt +++ b/here-naksha-lib-psql/src/commonMain/kotlin/naksha/psql/executors/query/WhereClauseBuilder.kt @@ -173,8 +173,8 @@ class WhereClauseBuilder(private val request: ReadFeatures) { } } - private tailrec fun whereNestedTags(tagQuery: ITagQuery){ - when(tagQuery){ + private tailrec fun whereNestedTags(tagQuery: ITagQuery) { + when (tagQuery) { is TagNot -> not(tagQuery.query, this::whereNestedTags) is TagOr -> or(tagQuery.filterNotNull(), this::whereNestedTags) is TagAnd -> and(tagQuery.filterNotNull(), this::whereNestedTags) @@ -182,40 +182,64 @@ class WhereClauseBuilder(private val request: ReadFeatures) { } } - private fun resolveSingleTagQuery(tagQuery: TagQuery){ - when(tagQuery){ + private fun resolveSingleTagQuery(tagQuery: TagQuery) { + when (tagQuery) { is TagExists -> { - where.append("$tagsAsJsonb ? ${tagQuery.name}") + where.append("$tagsAsJsonb ?? '${tagQuery.name}'") } + is TagValueIsNull -> { where.append("${tagValue(tagQuery)} = null") } + is TagValueIsBool -> { - if(tagQuery.value){ - where.append(tagValue(tagQuery)) + if (tagQuery.value) { + where.append(tagValue(tagQuery, PgType.BOOLEAN)) } else { - where.append("not(${tagValue(tagQuery)}})") + where.append("not(${tagValue(tagQuery, PgType.BOOLEAN)}})") } } + is TagValueIsDouble -> { val valuePlaceholder = placeholderForArg(tagQuery.value, PgType.DOUBLE) - val doubleOp = resolveDoubleOp(tagQuery.op, tagValue(tagQuery), valuePlaceholder) + val doubleOp = resolveDoubleOp( + tagQuery.op, + tagValue(tagQuery, PgType.DOUBLE), + valuePlaceholder + ) where.append(doubleOp) } + is TagValueIsString -> { val valuePlaceholder = placeholderForArg(tagQuery.value, PgType.STRING) - val stringEquals = resolveStringOp(StringOp.EQUALS, tagValue(tagQuery), valuePlaceholder) - where.append(stringEquals) + val stringEquals = resolveStringOp( + StringOp.EQUALS, + tagValue(tagQuery, PgType.STRING), + valuePlaceholder + ) + where.append(stringEquals) // naksha_tags(tags, flags)::jsonb->>foo = $1 } + is TagValueMatches -> { - val regexPlaceholder = placeholderForArg(tagQuery.regex, PgType.STRING) - where.append("$tagsAsJsonb @? '$[?(@.${tagQuery.name}=~/$regexPlaceholder/)]") + /* + SELECT * + FROM read_by_tags_test + WHERE naksha_tags(tags, flags) @? '$.year ? (@ like_regex "^202\\d$")' + */ +// val regex = Regex.escape(tagQuery.regex) + val regex = tagQuery.regex + where.append("$tagsAsJsonb @?? '\$.${tagQuery.name} ? (@ like_regex \"${regex}\")'") } } } - private fun tagValue(tagQuery: TagQuery): String = - "$tagsAsJsonb->>${tagQuery.name}" + private fun tagValue(tagQuery: TagQuery, castTo: PgType? = null): String { + return when (castTo) { + null -> "$tagsAsJsonb->'${tagQuery.name}'" + PgType.STRING -> "$tagsAsJsonb->>'${tagQuery.name}'" + else -> "($tagsAsJsonb->'${tagQuery.name}')::${castTo.value}" + } + } private fun not(subClause: T, subClauseResolver: (T) -> Unit) { where.append(" NOT (") @@ -282,8 +306,8 @@ class WhereClauseBuilder(private val request: ReadFeatures) { ) } } - + companion object { - private val tagsAsJsonb = "naksha_tags(${PgColumn.tags}, ${PgColumn.flags})::jsonb" + private val tagsAsJsonb = "naksha_tags(${PgColumn.tags}, ${PgColumn.flags})" } } diff --git a/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/ReadFeaturesByTagsTest.kt b/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/ReadFeaturesByTagsTest.kt index d927db3cd..3c72c10e8 100644 --- a/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/ReadFeaturesByTagsTest.kt +++ b/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/ReadFeaturesByTagsTest.kt @@ -5,27 +5,26 @@ import naksha.model.objects.NakshaCollection import naksha.model.objects.NakshaFeature import naksha.model.request.ReadFeatures import naksha.model.request.SuccessResponse -import naksha.model.request.query.TagExists -import naksha.model.request.query.TagQuery -import naksha.model.request.query.TagValueIsString +import naksha.model.request.query.* import naksha.psql.base.PgTestBase import naksha.psql.util.ProxyFeatureGenerator import kotlin.test.Test import kotlin.test.assertEquals +import kotlin.test.assertTrue class ReadFeaturesByTagsTest : PgTestBase(NakshaCollection("read_by_tags_test")) { @Test fun shouldReturnFeaturesWithExistingTag() { // Given: - val inputFeature = randomFeatureWithTags("foo=bar") + val inputFeature = randomFeatureWithTags("sample") // When: insertFeature(feature = inputFeature) // And: val featuresWithFooTag = executeTagsQuery( - TagExists("foo") + TagExists("sample") ).features // Then: @@ -36,14 +35,14 @@ class ReadFeaturesByTagsTest : PgTestBase(NakshaCollection("read_by_tags_test")) @Test fun shouldNotReturnFeaturesWithMissingTag() { // Given: - val inputFeature = randomFeatureWithTags("foo") + val inputFeature = randomFeatureWithTags("existing") // When: insertFeature(feature = inputFeature) // And: val featuresWithFooTag = executeTagsQuery( - TagExists("bar") + TagExists("non-existing") ).features // Then: @@ -51,7 +50,7 @@ class ReadFeaturesByTagsTest : PgTestBase(NakshaCollection("read_by_tags_test")) } @Test - fun shouldReturnFeaturesWithTagValue() { + fun shouldReturnFeaturesWithStringValue() { // Given: val inputFeature = randomFeatureWithTags("foo=bar") @@ -68,13 +67,157 @@ class ReadFeaturesByTagsTest : PgTestBase(NakshaCollection("read_by_tags_test")) assertEquals(inputFeature.id, featuresWithFooTag[0]!!.id) } + @Test + fun shouldReturnFeaturesByTagRegex() { + // Given: + val featureFrom2024 = randomFeatureWithTags("year=2024") + val featureFrom2030 = randomFeatureWithTags("year=2030") + + // When: + insertFeatures(featureFrom2024, featureFrom2030) + + // And: + val featuresFromThisDecade = executeTagsQuery( + TagValueMatches(name = "year", regex = "202[0-9]") + ).features + + // Then: + assertEquals(1, featuresFromThisDecade.size) + assertEquals(featureFrom2024.id, featuresFromThisDecade[0]!!.id) + } + + @Test + fun shouldReturnFeaturesWithDoubleValue() { + // Given: + val inputFeatures = listOf( + randomFeatureWithTags("some_number:=1").apply { id = "one" }, + randomFeatureWithTags("some_number:=5").apply { id = "five" }, + ) + + // When: + insertFeatures(inputFeatures) + + // And: + val featuresGt2 = executeTagsQuery( + TagValueIsDouble("some_number", DoubleOp.GT, 1.0) + ).features + + // Then: + assertEquals(1, featuresGt2.size) + assertEquals("five", featuresGt2[0]!!.id) + + // When + val featuresLte5 = executeTagsQuery( + TagValueIsDouble("some_number", DoubleOp.LTE, 5.0) + ).features + + // Then: + assertEquals(2, featuresLte5.size) + val lte5ids = featuresLte5.map { it!!.id } + assertTrue(lte5ids.containsAll(listOf("one", "five"))) + + // When: + val featuresEq6 = executeTagsQuery( + TagValueIsDouble("some_number", DoubleOp.EQ, 6.0) + ).features + + // Then: + assertTrue(featuresEq6.isEmpty()) + } + + @Test + fun shouldReturnFeaturesForComposedTagQuery() { + // Given: + val activeJohn = randomFeatureWithTags( + "username=john_doe", + "is_active:=true", + ) + val activeNick = randomFeatureWithTags( + "username=nick_foo", + "is_active:=true", + ) + val inactiveJohn = randomFeatureWithTags( + "username=john_bar", + "is_active:=false", + ) + val oldAdmin = randomFeatureWithTags( + "username=some_admin", + "role=admin" + ) + val invalidUserWithoutId = randomFeatureWithTags("is_active:=true") + + // And: + insertFeatures(activeJohn, activeNick, inactiveJohn, oldAdmin, invalidUserWithoutId) + + + // When: + val activeJohnsOrAdmin = TagOr( + TagAnd( + TagValueMatches(name = "username", regex = "john.+"), + TagValueIsBool(name = "is_active", value = true) + ), + TagValueIsString(name = "role", value = "admin") + ) + val features = executeTagsQuery(activeJohnsOrAdmin).features + + // Then: + assertEquals(2, features.size) + val featureIds = features.map { it!!.id } + featureIds.containsAll( + listOf( + activeJohn.id, + oldAdmin.id + ) + ) + } + + @Test + fun shouldTreatRefAsValueless() { + // Given: + val feature = randomFeatureWithTags("ref_lorem=ipsum") + insertFeatures(feature) + + // When + val byTagName = executeTagsQuery(TagExists("ref_lorem")).features + + // Then + assertTrue(byTagName.isEmpty()) + + // When + val byFullTag = executeTagsQuery(TagExists("ref_lorem=ipsum")).features + + // Then + assertEquals(1, byFullTag.size) + assertEquals(feature.id, byFullTag[0]!!.id) + } + + @Test + fun shouldTreatSourceIDAsValueless() { + // Given: + val feature = randomFeatureWithTags("sourceID:=123") + insertFeatures(feature) + + // When + val byTagName = executeTagsQuery(TagExists("sourceID")).features + + // Then + assertTrue(byTagName.isEmpty()) + + // When + val byFullTag = executeTagsQuery(TagExists("sourceID:=123")).features + + // Then + assertEquals(1, byFullTag.size) + assertEquals(feature.id, byFullTag[0]!!.id) + } + private fun randomFeatureWithTags(vararg tags: String): NakshaFeature { return ProxyFeatureGenerator.generateRandomFeature().apply { properties.xyz.tags = TagList(*tags) } } - private fun executeTagsQuery(tagQuery: TagQuery): SuccessResponse { + private fun executeTagsQuery(tagQuery: ITagQuery): SuccessResponse { return executeRead(ReadFeatures().apply { collectionIds += collection!!.id query.tags = tagQuery diff --git a/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/TagNormalizerTest.kt b/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/TagNormalizerTest.kt new file mode 100644 index 000000000..f28b1437c --- /dev/null +++ b/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/TagNormalizerTest.kt @@ -0,0 +1,92 @@ +package naksha.psql + +import naksha.model.TagNormalizer +import kotlin.test.Test +import kotlin.test.assertEquals + +class TagNormalizerTest { + + @Test + fun shouldRemoveNonAscii(){ + val tagsToBeClearedFromAscii = mapOf( + "p®¡©e=100£" to "pe=100", // regular tag + "~twice_¼=single_½" to "~twice_=single_", // starting with '~' + "#some=tµ¶ag" to "#some=tag", // starting with '#' + ) + + tagsToBeClearedFromAscii.forEach { (before, after) -> + assertEquals(after, TagNormalizer.normalizeTag(before)) + } + } + + @Test + fun shouldLeaveNonAsciiAsIs(){ + val tagsWithAsciiToBePreserved = listOf( + "@p®¡©e=100£", // starting with '@' + "ref_p®¡©e=100£", // starting with 'ref_' + "sourceIDp®¡©e=100£", // starting with 'sourceID' + ) + + tagsWithAsciiToBePreserved.forEach { tag -> + assertEquals(tag, TagNormalizer.normalizeTag(tag)) + } + } + + @Test + fun shouldLowercase(){ + val tag = "Some_Tag:=1235" + assertEquals(tag.lowercase(), TagNormalizer.normalizeTag(tag)) + } + + @Test + fun shouldNotLowercase(){ + val tagsNotToBeLowercased = listOf( + "@Some_Tag:=1235", + "ref_Some_Tag:=1235", + "~Some_Tag:=1235", + "#Some_Tag:=1235", + "sourceIDSome_Tag:=1235" + ) + + tagsNotToBeLowercased.forEach { tag -> + assertEquals(tag, TagNormalizer.normalizeTag(tag)) + } + } + + @Test + fun shouldSplit(){ + val tagsToBeSplit = listOf( + "@some_tag:=1235", + "~some_tag:=1235", + "#some_tag:=1235", + "some_tag:=1235" + ) + + tagsToBeSplit.forEach { rawTag -> + val expectedKey = rawTag.split(":")[0] + val normalized = TagNormalizer.normalizeTag(rawTag) + val tag = TagNormalizer.splitNormalizedTag(normalized) + + assertEquals(expectedKey, tag.key) + assertEquals(1235.0, tag.value) + assertEquals(rawTag, tag.tag) + } + } + + @Test + fun shouldNotSplit(){ + val tagsNotToBeSplit = listOf( + "ref_some_tag:=1235", + "sourceIDsome_tag:=1235" + ) + + tagsNotToBeSplit.forEach { rawTag -> + val normalized = TagNormalizer.normalizeTag(rawTag) + val tag = TagNormalizer.splitNormalizedTag(normalized) + + assertEquals(rawTag, tag.key) + assertEquals(null, tag.value) + assertEquals(rawTag, tag.tag) + } + } +} \ No newline at end of file diff --git a/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/base/PgTestBase.kt b/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/base/PgTestBase.kt index 23fef9fc2..ed7b1e6d1 100644 --- a/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/base/PgTestBase.kt +++ b/here-naksha-lib-psql/src/commonTest/kotlin/naksha/psql/base/PgTestBase.kt @@ -34,6 +34,9 @@ abstract class PgTestBase(val collection: NakshaCollection? = null) { protected fun insertFeature(feature: NakshaFeature, sessionOptions: SessionOptions? = null) = insertFeatures(listOf(feature), sessionOptions) + protected fun insertFeatures(vararg features: NakshaFeature) = + insertFeatures(listOf(*features)) + protected fun insertFeatures( features: List, sessionOptions: SessionOptions? = null diff --git a/here-naksha-lib-psql/src/jvmMain/kotlin/naksha/psql/PsqlPlan.kt b/here-naksha-lib-psql/src/jvmMain/kotlin/naksha/psql/PsqlPlan.kt index 5def357be..5f678d545 100644 --- a/here-naksha-lib-psql/src/jvmMain/kotlin/naksha/psql/PsqlPlan.kt +++ b/here-naksha-lib-psql/src/jvmMain/kotlin/naksha/psql/PsqlPlan.kt @@ -21,6 +21,13 @@ class PsqlPlan(internal val query: PsqlQuery, conn: Connection) : PgPlan { stmt.execute() return PsqlCursor(stmt, false) } + /** + * TODO: remove + * + * SELECT gzip(bytea_agg(tuple_number)) AS rs FROM (SELECT tuple_number FROM ( + * (SELECT tuple_number, id FROM read_by_tags_test WHERE (naksha_tags(tags, flags)::jsonb->>foo = '''bar''')) + * ) ORDER BY id, tuple_number) LIMIT 1000000 + */ /** * Adds the prepared statement with the given arguments into batch-execution queue. This requires a mutation query like UPDATE or diff --git a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorage.java b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorage.java index e8ab40084..7b48db5ad 100644 --- a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorage.java +++ b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorage.java @@ -20,15 +20,15 @@ import static com.here.naksha.storage.http.RequestSender.KeyProperties; -import naksha.model.NakshaContext; import com.here.naksha.lib.core.lambdas.Fe1; import com.here.naksha.lib.core.models.naksha.Storage; -import naksha.model.IReadSession; -import naksha.model.IStorage; import com.here.naksha.lib.core.util.json.JsonSerializable; import com.here.naksha.storage.http.cache.RequestSenderCache; import java.util.concurrent.Future; import java.util.concurrent.FutureTask; +import naksha.model.IReadSession; +import naksha.model.IStorage; +import naksha.model.NakshaContext; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import org.slf4j.Logger; diff --git a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageProperties.java b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageProperties.java index 9e58f1d87..cc5d6cda6 100644 --- a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageProperties.java +++ b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageProperties.java @@ -20,9 +20,9 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import naksha.model.NakshaVersion; -import naksha.geo.XyzProperties; import java.util.Map; +import naksha.geo.XyzProperties; +import naksha.model.NakshaVersion; import org.jetbrains.annotations.ApiStatus.AvailableSince; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; diff --git a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageReadExecute.java b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageReadExecute.java index d200a3a4c..929a0c359 100644 --- a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageReadExecute.java +++ b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageReadExecute.java @@ -22,12 +22,7 @@ import static com.here.naksha.storage.http.PrepareResult.prepareResult; import static java.lang.String.format; -import naksha.model.NakshaContext; import com.here.naksha.lib.core.models.XyzError; -import naksha.model.XyzFeature; -import naksha.model.XyzFeatureCollection; -import naksha.model.ErrorResult; -import naksha.model.POp; import com.here.naksha.lib.core.models.storage.ReadFeaturesProxyWrapper; import com.here.naksha.lib.core.models.storage.Result; import java.net.HttpURLConnection; @@ -37,6 +32,11 @@ import java.util.List; import java.util.Map; import java.util.stream.Collectors; +import naksha.model.ErrorResult; +import naksha.model.NakshaContext; +import naksha.model.POp; +import naksha.model.XyzFeature; +import naksha.model.XyzFeatureCollection; import org.jetbrains.annotations.NotNull; import org.slf4j.Logger; import org.slf4j.LoggerFactory; diff --git a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageReadSession.java b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageReadSession.java index e77a8b943..e87840105 100644 --- a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageReadSession.java +++ b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/HttpStorageReadSession.java @@ -18,14 +18,13 @@ */ package com.here.naksha.storage.http; -import naksha.model.NakshaContext; import com.here.naksha.lib.core.models.XyzError; import com.here.naksha.lib.core.models.storage.*; -import naksha.model.IReadSession; import java.util.concurrent.TimeUnit; - -import naksha.model.ReadRequest; import naksha.model.ErrorResult; +import naksha.model.IReadSession; +import naksha.model.NakshaContext; +import naksha.model.ReadRequest; import org.apache.commons.lang3.NotImplementedException; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; diff --git a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/POpToQueryConverter.java b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/POpToQueryConverter.java index c31ece3f9..0530899c8 100644 --- a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/POpToQueryConverter.java +++ b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/POpToQueryConverter.java @@ -21,14 +21,14 @@ import static com.here.naksha.lib.core.exceptions.UncheckedException.unchecked; import static naksha.model.POpType.*; -import naksha.model.POp; -import naksha.model.POpType; import java.net.URLEncoder; import java.nio.charset.StandardCharsets; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.stream.Collectors; +import naksha.model.POp; +import naksha.model.POpType; import org.apache.commons.lang3.ArrayUtils; import org.jetbrains.annotations.NotNull; diff --git a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/PrepareResult.java b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/PrepareResult.java index 5083a44fa..3b4a87658 100644 --- a/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/PrepareResult.java +++ b/here-naksha-storage-http/src/main/java/com/here/naksha/storage/http/PrepareResult.java @@ -22,7 +22,6 @@ import com.here.naksha.lib.core.models.Typed; import com.here.naksha.lib.core.models.XyzError; -import naksha.model.XyzFeature; import com.here.naksha.lib.core.models.storage.*; import com.here.naksha.lib.core.util.json.JsonSerializable; import java.io.ByteArrayInputStream; @@ -35,8 +34,8 @@ import java.util.List; import java.util.function.Function; import java.util.zip.GZIPInputStream; - import naksha.model.ErrorResult; +import naksha.model.XyzFeature; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable;