From 335977f229f663c6324e40bc0c1b8759a10e572e Mon Sep 17 00:00:00 2001 From: Valentyn Sobol Date: Tue, 29 Oct 2024 16:42:22 +0300 Subject: [PATCH] Storage optimization (#274) [jacodb-core] Add de-duplication of attribute values in AttributesImmutable As of this commit, values array contains only different attribute values. [jacodb-core] Close ers iff it was initialized [jacodb-core] Processed only jar files referenced in the Class-Path attribute of manifest [jacodb-storage] Fix minor memory usage issue in AttributesImmutable Avoid using class constructor parameters in lazy fields. [jacodb-storage] Reduce memory usage by InstanceIdCollections In this commit, two implementation of InstanceIdCollection added, SortedIntArrayInstanceIdCollection & UnsortedIntArrayInstanceIdCollection. If instances ids of particular attributes cannot be packed in a LongRange, but if they all can be represented as an Int, then corresponding implementations wrap IntArray, not LongArray. [jacodb-storage] Minor: toAttributesImmutable() is an extension for list of attribute pairs [jacodb-storage] Optimize AttributesImmutable.navigate() in case of sorted by value attributes If attributes are sorted by value, in binary search algorithm, getting instance id by index of an attribute value can be skipped. In that case, copying of the attribute value in an extra byte array is redundant, and comparing can be done by a special comparison function. [jacodb-storage] Re-order built-in bindings [jacodb-storage] Optimize dealing with instance ids in AttributesImmutable Instead of LongArray, instance ids are represented by implementors of InstanceIdCollection. LongRangeInstanceIdCollection saves memory by using LongRange instead of LongArray. The getByIndex() method introduced and used where possible in order to skip conversion of instance id to index in InstanceIdCollection to load values from values byte array. [jacodb-benchmarks] Make RAMEntityRelationshipStorageBenchmarks more correct Fewer operations on each invocation, this reduces Java GC load. [jacodb-storage] Optimize PropertiesMutable.getEntitiesLtValue() & PropertiesMutable.getEntitiesEqOrLtValue() Minor optimization: exit iteration over value index earlier. This is okay since all further values are greater than specified one. [jacodb-benchmarks] Add benchmarks of RAM implementation of ERS API [jacodb-storage] Get rid of using one-nio library ByteArrayBuilder added instead of one-nio's one. SparseBitSet moved to the 'org.jacodb.util.collections' package. --- jacodb-benchmarks/build.gradle.kts | 5 + .../RAMEntityRelationshipStorageBenchmarks.kt | 166 ++++++++++++++++ .../org/jacodb/impl/fs/ByteCodeLocations.kt | 8 +- .../impl/storage/SQLitePersistenceImpl.kt | 10 +- jacodb-storage/build.gradle.kts | 1 + .../impl/storage/ers/BuiltInBindings.kt | 6 +- .../storage/ers/ram/AttributesImmutable.kt | 184 +++++++++++++++--- .../org/jacodb/impl/storage/ers/ram/Links.kt | 2 +- .../jacodb/impl/storage/ers/ram/Properties.kt | 6 +- .../ers/ram/RAMDataContainerImmutable.kt | 1 + .../ers/ram/RAMDataContainerMutable.kt | 4 +- .../org/jacodb/util/ByteArrayBuilder.kt | 44 +++++ .../ram => util/collections}/SparseBitSet.kt | 3 +- .../collections}/SparseBitSetTest.kt | 2 +- 14 files changed, 398 insertions(+), 44 deletions(-) create mode 100644 jacodb-benchmarks/src/test/kotlin/org/jacodb/testing/performance/ers/RAMEntityRelationshipStorageBenchmarks.kt create mode 100644 jacodb-storage/src/main/kotlin/org/jacodb/util/ByteArrayBuilder.kt rename jacodb-storage/src/main/kotlin/org/jacodb/{impl/storage/ers/ram => util/collections}/SparseBitSet.kt (96%) rename jacodb-storage/src/test/kotlin/org/jacodb/{impl/storage/ers/ram => util/collections}/SparseBitSetTest.kt (98%) diff --git a/jacodb-benchmarks/build.gradle.kts b/jacodb-benchmarks/build.gradle.kts index 39f7aa962..036aee05d 100644 --- a/jacodb-benchmarks/build.gradle.kts +++ b/jacodb-benchmarks/build.gradle.kts @@ -9,6 +9,7 @@ plugins { dependencies { implementation(project(":jacodb-core")) + implementation(project(":jacodb-storage")) implementation(testFixtures(project(":jacodb-core"))) implementation(Libs.kotlin_logging) @@ -79,6 +80,10 @@ benchmark { include("GuavaCacheBenchmarks") include("XodusCacheBenchmarks") } + register("ersRam") { + include("RAMEntityRelationshipStorageMutableBenchmarks") + include("RAMEntityRelationshipStorageImmutableBenchmarks") + } } } diff --git a/jacodb-benchmarks/src/test/kotlin/org/jacodb/testing/performance/ers/RAMEntityRelationshipStorageBenchmarks.kt b/jacodb-benchmarks/src/test/kotlin/org/jacodb/testing/performance/ers/RAMEntityRelationshipStorageBenchmarks.kt new file mode 100644 index 000000000..5fbde9bc9 --- /dev/null +++ b/jacodb-benchmarks/src/test/kotlin/org/jacodb/testing/performance/ers/RAMEntityRelationshipStorageBenchmarks.kt @@ -0,0 +1,166 @@ +/* + * Copyright 2022 UnitTestBot contributors (utbot.org) + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jacodb.testing.performance.ers + +import org.jacodb.api.storage.ers.EmptyErsSettings +import org.jacodb.api.storage.ers.Entity +import org.jacodb.api.storage.ers.EntityId +import org.jacodb.api.storage.ers.EntityRelationshipStorage +import org.jacodb.api.storage.ers.EntityRelationshipStorageSPI +import org.jacodb.api.storage.ers.Transaction +import org.jacodb.impl.storage.ers.ram.RAM_ERS_SPI +import org.junit.jupiter.api.AfterEach +import org.junit.jupiter.api.BeforeEach +import org.junit.jupiter.api.Test +import org.openjdk.jmh.annotations.Benchmark +import org.openjdk.jmh.annotations.BenchmarkMode +import org.openjdk.jmh.annotations.Fork +import org.openjdk.jmh.annotations.Level +import org.openjdk.jmh.annotations.Measurement +import org.openjdk.jmh.annotations.Mode +import org.openjdk.jmh.annotations.OutputTimeUnit +import org.openjdk.jmh.annotations.Scope +import org.openjdk.jmh.annotations.Setup +import org.openjdk.jmh.annotations.State +import org.openjdk.jmh.annotations.TearDown +import org.openjdk.jmh.annotations.Warmup +import org.openjdk.jmh.infra.Blackhole +import java.util.concurrent.TimeUnit + +abstract class RAMEntityRelationshipStorageBenchmarks { + + private val ersSpi by lazy(LazyThreadSafetyMode.NONE) { + EntityRelationshipStorageSPI.getProvider(RAM_ERS_SPI) + } + protected lateinit var storage: EntityRelationshipStorage + private lateinit var txn: Transaction + private lateinit var entity: Entity + private lateinit var loginSearchValue: String + private lateinit var passwordSearchValue: String + + @Benchmark + @OutputTimeUnit(TimeUnit.NANOSECONDS) + fun getLogin(hole: Blackhole) { + hole.consume(entity.getRawProperty("login")) + } + + @Benchmark + @OutputTimeUnit(TimeUnit.NANOSECONDS) + fun getPassword(hole: Blackhole) { + hole.consume(entity.getRawProperty("password")) + } + + @Benchmark + @OutputTimeUnit(TimeUnit.MICROSECONDS) + fun findByLogin(hole: Blackhole) { + hole.consume(txn.find("User", "login", loginSearchValue).first()) + } + + @Benchmark + @OutputTimeUnit(TimeUnit.MICROSECONDS) + fun findByPassword(hole: Blackhole) { + hole.consume(txn.find("User", "password", passwordSearchValue).first()) + } + + @Benchmark + @OutputTimeUnit(TimeUnit.MICROSECONDS) + fun findLt(hole: Blackhole) { + hole.consume(txn.findLt("User", "age", 50).first()) + } + + @Benchmark + @OutputTimeUnit(TimeUnit.MICROSECONDS) + fun findEqOrLt(hole: Blackhole) { + hole.consume(txn.findEqOrLt("User", "age", 50).first()) + } + + @Benchmark + @OutputTimeUnit(TimeUnit.MICROSECONDS) + fun findGt(hole: Blackhole) { + hole.consume(txn.findGt("User", "age", 50).first()) + } + + @Benchmark + @OutputTimeUnit(TimeUnit.MICROSECONDS) + fun findEqOrGt(hole: Blackhole) { + hole.consume(txn.findEqOrGt("User", "age", 50).first()) + } + + @Setup(Level.Iteration) + fun setupIteration() { + createMutable() + populate() + setImmutable() + txn = storage.beginTransaction(readonly = true) + entity = txn.getEntityUnsafe(EntityId(0, (Math.random() * 1_000_000).toLong())) + loginSearchValue = "login${entity.id.instanceId}" + passwordSearchValue = "a very secure password ${entity.id.instanceId}" + } + + @TearDown(Level.Iteration) + fun tearDownIteration() { + txn.abort() + storage.close() + } + + @Setup(Level.Invocation) + fun setupInvocation() { + loginSearchValue = "login${entity.id.instanceId}" + passwordSearchValue = "a very secure password ${entity.id.instanceId}" + } + + private fun createMutable() { + storage = ersSpi.newStorage(null, EmptyErsSettings) + } + + private fun populate() { + storage.transactional { txn -> + repeat(1_000_000) { i -> + val user = txn.newEntity("User") + user["login"] = "login$i" + user["password"] = "a very secure password $i" + user["age"] = 20 + i % 80 + } + } + } + + abstract fun setImmutable() +} + +@State(Scope.Benchmark) +@Fork(1, jvmArgs = ["-Xmx8g", "-Xms8g"]) +@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +@BenchmarkMode(Mode.AverageTime) +@Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS) +class RAMEntityRelationshipStorageMutableBenchmarks : RAMEntityRelationshipStorageBenchmarks() { + + override fun setImmutable() { + // do nothing im mutable benchmark + } +} + +@State(Scope.Benchmark) +@Fork(1, jvmArgs = ["-Xmx8g", "-Xms8g"]) +@Warmup(iterations = 5, time = 1, timeUnit = TimeUnit.SECONDS) +@BenchmarkMode(Mode.AverageTime) +@Measurement(iterations = 10, time = 1, timeUnit = TimeUnit.SECONDS) +class RAMEntityRelationshipStorageImmutableBenchmarks : RAMEntityRelationshipStorageBenchmarks() { + + override fun setImmutable() { + storage = storage.asReadonly + } +} \ No newline at end of file diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/fs/ByteCodeLocations.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/fs/ByteCodeLocations.kt index a7364e5b9..b0f174ed7 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/fs/ByteCodeLocations.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/fs/ByteCodeLocations.kt @@ -35,7 +35,7 @@ fun File.asByteCodeLocation(runtimeVersion: JavaVersion, isRuntime: Boolean = fa if (!exists()) { throw IllegalArgumentException("file $absolutePath doesn't exist") } - if (isFile && name.endsWith(".jar") || name.endsWith(".jmod")) { + if (isJar()) { return mutableSetOf().also { classPath(it) }.map { JarLocation(it, isRuntime, runtimeVersion) } } else if (isDirectory) { return listOf(BuildFolderLocation(this)) @@ -52,7 +52,7 @@ fun Collection.filterExisting(): List = filter { file -> } private fun File.classPath(classpath: MutableCollection) { - if (exists() && classpath.add(this)) { + if (isJar() && exists() && classpath.add(this)) { JarFile(this).use { jarFile -> jarFile.manifest?.mainAttributes?.getValue("Class-Path")?.split(' ')?.forEach { ref -> Paths.get( @@ -61,4 +61,6 @@ private fun File.classPath(classpath: MutableCollection) { } } } -} \ No newline at end of file +} + +private fun File.isJar() = isFile && name.endsWith(".jar") || name.endsWith(".jmod") diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/SQLitePersistenceImpl.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/SQLitePersistenceImpl.kt index 16c6a7164..2aca922a3 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/SQLitePersistenceImpl.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/SQLitePersistenceImpl.kt @@ -51,11 +51,15 @@ class SQLitePersistenceImpl( internal val jooq = DSL.using(connection, SQLDialect.SQLITE, Settings().withExecuteLogging(false)) private val lock = ReentrantLock() private val persistenceService = SQLitePersistenceService(this) + private var ersInitialized = false + override val ers: EntityRelationshipStorage by lazy { SqlEntityRelationshipStorage( dataSource, BuiltInBindingProvider - ) + ).also { + ersInitialized = true + } } companion object { @@ -81,7 +85,9 @@ class SQLitePersistenceImpl( override fun close() { try { - ers.close() + if (ersInitialized) { + ers.close() + } connection.close() super.close() } catch (e: Exception) { diff --git a/jacodb-storage/build.gradle.kts b/jacodb-storage/build.gradle.kts index c09c59312..f4e287350 100644 --- a/jacodb-storage/build.gradle.kts +++ b/jacodb-storage/build.gradle.kts @@ -5,6 +5,7 @@ dependencies { compileOnly(Libs.lmdb_java) compileOnly(Libs.rocks_db) + testImplementation(Libs.xodusEnvironment) testImplementation(Libs.lmdb_java) testImplementation(Libs.rocks_db) diff --git a/jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/ers/BuiltInBindings.kt b/jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/ers/BuiltInBindings.kt index ce6b509b1..d3e3a898c 100644 --- a/jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/ers/BuiltInBindings.kt +++ b/jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/ers/BuiltInBindings.kt @@ -35,12 +35,12 @@ fun Class.getBinding(): Binding = (when { fun getBinding(obj: T): Binding = obj.javaClass.getBinding() private val builtInBindings: Array> = arrayOf( - ByteArrayBinding, // trivial binding with no conversion to make it possible to deal with ByteArray properties StringBinding, // UTF-8 strings - IntegerBinding, // 4-byte signed integers LongBinding, // 8-byte signed integers (longs) + IntegerBinding, // 4-byte signed integers + BooleanBinding, // boolean values DoubleBinging, // 8-byte floating point numbers (doubles) - BooleanBinding // boolean values + ByteArrayBinding // trivial binding with no conversion to make it possible to deal with ByteArray properties ) private abstract class BuiltInBinding : Binding { diff --git a/jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/ers/ram/AttributesImmutable.kt b/jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/ers/ram/AttributesImmutable.kt index 4b8219350..60b46e5a3 100644 --- a/jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/ers/ram/AttributesImmutable.kt +++ b/jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/ers/ram/AttributesImmutable.kt @@ -21,45 +21,55 @@ import org.jacodb.api.storage.asComparable import org.jacodb.api.storage.ers.Entity import org.jacodb.api.storage.ers.EntityId import org.jacodb.api.storage.ers.EntityIterable +import org.jacodb.util.ByteArrayBuilder +import kotlin.math.min -internal fun Any.toAttributesImmutable(instanceValues: List>): AttributesImmutable { - if (instanceValues.isEmpty()) { +internal fun List>.toAttributesImmutable(): AttributesImmutable { + if (isEmpty()) { return EmptyAttributesImmutable } - val totalSize = instanceValues.fold(0) { sum, instanceValue -> sum + instanceValue.second.size } - val values = ByteArray(totalSize) - val instanceIds = LongArray(instanceValues.size) - val offsetAndLens = LongArray(instanceValues.size) + val values = ByteArrayBuilder() + val instanceIds = LongArray(size) + val offsetAndLens = LongArray(size) + val differentValues = hashMapOf() + var offset = 0 - instanceValues.forEachIndexed { i, (instanceId, value) -> - val len = value.size - value.copyInto(destination = values, destinationOffset = offset) - val indexValue = (len.toLong() shl 32) + offset + forEachIndexed { i, (instanceId, value) -> instanceIds[i] = instanceId + val valueKey = value.asComparable() + var indexValue = differentValues[valueKey] + if (indexValue == null) { + val len = value.size + values.append(value) + indexValue = (len.toLong() shl 32) + offset + differentValues[valueKey] = indexValue + offset += len + } offsetAndLens[i] = indexValue - offset += len } - return AttributesImmutable(values, instanceIds, offsetAndLens) + return AttributesImmutable(values.toByteArray(), instanceIds, offsetAndLens) } internal open class AttributesImmutable( private val values: ByteArray, - private val instanceIds: LongArray, + instanceIds: LongArray, private val offsetAndLens: LongArray ) { - private val sameOrder: Boolean // `true` if order of instance ids is the same as the one sorted by value + private val instanceIdCollection = instanceIds.toInstanceIdCollection(sorted = true) + private val sortedByValue: Boolean // `true` if order of instance ids is the same as the one sorted by value private val sortedByValueInstanceIds by lazy { // NB! // We need stable sorting here, and java.util.Collections.sort() guarantees the sort is stable - instanceIds.sortedBy { get(it)!!.asComparable() }.toLongArray() + instanceIdCollection.asIterable() + .sortedBy { get(it)!!.asComparable() }.toLongArray().toInstanceIdCollection(sorted = false) } init { - var sameOrder = true + var sortedByValue = true var prevId = Long.MIN_VALUE var prevValue: ByteArrayKey? = null for (i in instanceIds.indices) { @@ -70,35 +80,32 @@ internal open class AttributesImmutable( } prevId = currentId // check if order of values is the same as order of ids - if (sameOrder) { - val currentValue = ByteArrayKey(get(currentId)!!) + if (sortedByValue) { + val currentValue = ByteArrayKey(getByIndex(i)) prevValue?.let { if (it > currentValue) { - sameOrder = false + sortedByValue = false } } prevValue = currentValue } } - this.sameOrder = sameOrder + this.sortedByValue = sortedByValue } operator fun get(instanceId: Long): ByteArray? { - val index = instanceIds.binarySearch(instanceId) + val index = instanceIdCollection.getIndex(instanceId) if (index < 0) { return null } - val offsetAndLen = offsetAndLens[index] - val offset = offsetAndLen.toInt() - val len = (offsetAndLen shr 32).toInt() - return values.sliceArray(offset until offset + len) + return getByIndex(index) } fun navigate(value: ByteArray, leftBound: Boolean): AttributesCursor { - if (instanceIds.isEmpty()) { + if (instanceIdCollection.isEmpty) { return EmptyAttributesCursor } - val ids = if (sameOrder) instanceIds else sortedByValueInstanceIds + val ids = if (sortedByValue) instanceIdCollection else sortedByValueInstanceIds val valueComparable = value.asComparable() // in order to find exact left or right bound, we have to use binary search without early break on equality var low = 0 @@ -106,8 +113,14 @@ internal open class AttributesImmutable( var found = -1 while (low <= high) { val mid = (low + high).ushr(1) - val midValue = get(ids[mid])!!.asComparable() - val cmp = valueComparable.compareTo(midValue) + val cmp = if (sortedByValue) { + val offsetAndLen = offsetAndLens[mid] + val offset = offsetAndLen.toInt() + val len = (offsetAndLen shr 32).toInt() + -compareValueTo(offset, len, value) + } else { + valueComparable.compareTo(get(ids[mid])!!.asComparable()) + } if (cmp == 0) { found = mid } @@ -125,7 +138,7 @@ internal open class AttributesImmutable( } } } - val index = if (found in ids.indices) found else -(low + 1) + val index = if (found in 0 until ids.size) found else -(low + 1) return object : AttributesCursor { private var idx: Int = if (index < 0) -index - 1 else index @@ -135,7 +148,7 @@ internal open class AttributesImmutable( override val current: Pair get() { val instanceId = ids[idx] - return instanceId to get(instanceId)!! + return instanceId to if (sortedByValue) getByIndex(idx) else get(instanceId)!! } override fun moveNext(): Boolean = ++idx < ids.size @@ -143,6 +156,24 @@ internal open class AttributesImmutable( override fun movePrev(): Boolean = --idx >= 0 } } + + private fun getByIndex(index: Int): ByteArray { + val offsetAndLen = offsetAndLens[index] + val offset = offsetAndLen.toInt() + val len = (offsetAndLen shr 32).toInt() + return values.sliceArray(offset until offset + len) + } + + /** + * Compare a value from values array identified by offset in the array and length of the value + */ + private fun compareValueTo(offset: Int, len: Int, other: ByteArray): Int { + for (i in 0 until min(len, other.size)) { + val cmp = (values[offset + i].toInt() and 0xff).compareTo(other[i].toInt() and 0xff) + if (cmp != 0) return cmp + } + return len - other.size + } } private object EmptyAttributesImmutable : AttributesImmutable(byteArrayOf(), longArrayOf(), longArrayOf()) @@ -207,4 +238,95 @@ internal class AttributesCursorEntityIterable( return next } } +} + +// Collection of instanceIds +private interface InstanceIdCollection { + val isEmpty: Boolean get() = size == 0 + val size: Int + operator fun get(index: Int): Long + fun getIndex(instanceId: Long): Int +} + +private fun LongArray.toInstanceIdCollection(sorted: Boolean): InstanceIdCollection { + if (isEmpty()) { + return EmptyInstanceIdCollection + } + if (sorted && this[0] == 0L && this[size - 1] == (size - 1).toLong()) { + return LongRangeInstanceIdCollection(0L until size) + } + return if (sorted) { + if (allInts()) { + SortedIntArrayInstanceIdCollection(toIntArray()) + } else { + SortedLongArrayInstanceIdCollection(this) + } + } else if (allInts()) { + UnsortedIntArrayInstanceIdCollection(toIntArray()) + } else { + UnsortedLongArrayInstanceIdCollection(this) + } +} + +private object EmptyInstanceIdCollection : InstanceIdCollection { + override val size = 0 + override fun get(index: Int) = error("Can't get in EmptyInstanceIdCollection") + override fun getIndex(instanceId: Long): Int = -1 +} + +// InstanceIdCollection wrapping unsorted LongArray +private class UnsortedLongArrayInstanceIdCollection(val array: LongArray) : InstanceIdCollection { + override val size: Int get() = array.size + override fun get(index: Int): Long = array[index] + override fun getIndex(instanceId: Long): Int = array.indexOf(instanceId) +} + +// InstanceIdCollection wrapping sorted LongArray +private class SortedLongArrayInstanceIdCollection(val array: LongArray) : InstanceIdCollection { + override val size: Int get() = array.size + override fun get(index: Int): Long = array[index] + override fun getIndex(instanceId: Long): Int = array.binarySearch(instanceId) +} + +// InstanceIdCollection wrapping LongRange which is growing progression with step 1 +private class LongRangeInstanceIdCollection(val range: LongRange) : InstanceIdCollection { + override val size: Int get() = (range.last - range.first).toInt() + 1 + override fun get(index: Int): Long = range.first + index + override fun getIndex(instanceId: Long): Int = (instanceId - range.first).toInt() +} + +// InstanceIdCollection wrapping unsorted IntArray +private class UnsortedIntArrayInstanceIdCollection(val array: IntArray) : InstanceIdCollection { + override val size: Int get() = array.size + override fun get(index: Int): Long = array[index].toLong() + override fun getIndex(instanceId: Long): Int = if (instanceId.isInt()) array.indexOf(instanceId.toInt()) else -1 +} + +// InstanceIdCollection wrapping sorted LongArray +private class SortedIntArrayInstanceIdCollection(val array: IntArray) : InstanceIdCollection { + override val size: Int get() = array.size + override fun get(index: Int): Long = array[index].toLong() + override fun getIndex(instanceId: Long): Int = + if (instanceId.isInt()) array.binarySearch(instanceId.toInt()) else -1 +} + +private fun Long.isInt() = this in 0L..Int.MAX_VALUE + +private fun LongArray.allInts(): Boolean { + return all { it.isInt() } +} + +private fun LongArray.toIntArray(): IntArray { + return IntArray(size) { i -> this[i].toInt() } +} + +private fun InstanceIdCollection.asIterable(): Iterable { + return when (this) { + is LongRangeInstanceIdCollection -> range + is SortedLongArrayInstanceIdCollection -> array.asIterable() + is UnsortedLongArrayInstanceIdCollection -> array.asIterable() + is SortedIntArrayInstanceIdCollection -> array.map { it.toLong() } + is UnsortedIntArrayInstanceIdCollection -> array.map { it.toLong() } + else -> error("Unknown InstanceIdCollection class: $javaClass") + } } \ No newline at end of file diff --git a/jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/ers/ram/Links.kt b/jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/ers/ram/Links.kt index 6424818b2..983d5fba0 100644 --- a/jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/ers/ram/Links.kt +++ b/jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/ers/ram/Links.kt @@ -122,7 +122,7 @@ internal fun LinksMutable.toImmutable(): LinksImmutable { } linkList += instanceId to valueArray } - return LinksImmutable(targetTypeId, toAttributesImmutable(linkList)) + return LinksImmutable(targetTypeId, linkList.toAttributesImmutable()) } /** diff --git a/jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/ers/ram/Properties.kt b/jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/ers/ram/Properties.kt index ddf99f37d..c8f846b4a 100644 --- a/jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/ers/ram/Properties.kt +++ b/jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/ers/ram/Properties.kt @@ -118,6 +118,8 @@ internal class PropertiesMutable( actualIndex.beginRead().iterator().forEach { if (it.key < bound) { result.addAll(it.value) + } else { + return@forEach } } return newProperties to InstanceIdCollectionEntityIterable( @@ -140,6 +142,8 @@ internal class PropertiesMutable( actualIndex.beginRead().iterator().forEach { if (it.key <= bound) { result.addAll(it.value) + } else { + return@forEach } } return newProperties to InstanceIdCollectionEntityIterable( @@ -278,5 +282,5 @@ internal class PropertiesImmutable(private val attributes: AttributesImmutable) } internal fun PropertiesMutable.toImmutable(): PropertiesImmutable { - return PropertiesImmutable(toAttributesImmutable(props.beginRead().map { it.key to it.value })) + return PropertiesImmutable(props.beginRead().map { it.key to it.value }.toAttributesImmutable()) } diff --git a/jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/ers/ram/RAMDataContainerImmutable.kt b/jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/ers/ram/RAMDataContainerImmutable.kt index db37810b4..54bedf643 100644 --- a/jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/ers/ram/RAMDataContainerImmutable.kt +++ b/jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/ers/ram/RAMDataContainerImmutable.kt @@ -19,6 +19,7 @@ package org.jacodb.impl.storage.ers.ram import org.jacodb.api.storage.ers.EntityId import org.jacodb.api.storage.ers.EntityIterable import org.jacodb.api.storage.ers.longRangeIterable +import org.jacodb.util.collections.SparseBitSet internal class RAMDataContainerImmutable( // map of entity types to their type ids diff --git a/jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/ers/ram/RAMDataContainerMutable.kt b/jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/ers/ram/RAMDataContainerMutable.kt index c26545722..937f6e92e 100644 --- a/jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/ers/ram/RAMDataContainerMutable.kt +++ b/jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/ers/ram/RAMDataContainerMutable.kt @@ -20,6 +20,8 @@ import org.jacodb.api.storage.ers.EntityId import org.jacodb.api.storage.ers.EntityIterable import org.jacodb.api.storage.ers.filterInstanceIds import org.jacodb.api.storage.ers.longRangeIterable +import org.jacodb.util.collections.EmptySparseBitSet +import org.jacodb.util.collections.SparseBitSet internal class RAMDataContainerMutable( private var typeIdCounter: Int, // next free type id @@ -105,7 +107,7 @@ internal class RAMDataContainerMutable( } val blobs = HashMap().also { map -> this.blobs.entries().forEach { entry -> - map[entry.key] = toAttributesImmutable(entry.value.entries().map { it.key to it.value }) + map[entry.key] = entry.value.entries().map { it.key to it.value }.toAttributesImmutable() } } return RAMDataContainerImmutable( diff --git a/jacodb-storage/src/main/kotlin/org/jacodb/util/ByteArrayBuilder.kt b/jacodb-storage/src/main/kotlin/org/jacodb/util/ByteArrayBuilder.kt new file mode 100644 index 000000000..e6d4922ab --- /dev/null +++ b/jacodb-storage/src/main/kotlin/org/jacodb/util/ByteArrayBuilder.kt @@ -0,0 +1,44 @@ +/* + * Copyright 2022 UnitTestBot contributors (utbot.org) + *

+ * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + *

+ * http://www.apache.org/licenses/LICENSE-2.0 + *

+ * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.jacodb.util + +import kotlin.math.max + +class ByteArrayBuilder(initialCapacity: Int = 1024) { + + private var buffer = ByteArray(initialCapacity) + private var count = 0 + + fun append(data: ByteArray): ByteArrayBuilder { + val len = data.size + ensureCapacity(count + len) + System.arraycopy(data, 0, buffer, count, len) + count += len + return this + } + + fun toByteArray(): ByteArray { + return if (buffer.size == count) buffer else buffer.copyOf(count) + } + + private fun ensureCapacity(minCapacity: Int) { + val capacity = buffer.size + if (capacity < minCapacity) { + buffer = buffer.copyOf(max(minCapacity, capacity * 2)) + } + } +} \ No newline at end of file diff --git a/jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/ers/ram/SparseBitSet.kt b/jacodb-storage/src/main/kotlin/org/jacodb/util/collections/SparseBitSet.kt similarity index 96% rename from jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/ers/ram/SparseBitSet.kt rename to jacodb-storage/src/main/kotlin/org/jacodb/util/collections/SparseBitSet.kt index 4f12325e6..3c0cf629c 100644 --- a/jacodb-storage/src/main/kotlin/org/jacodb/impl/storage/ers/ram/SparseBitSet.kt +++ b/jacodb-storage/src/main/kotlin/org/jacodb/util/collections/SparseBitSet.kt @@ -14,8 +14,9 @@ * limitations under the License. */ -package org.jacodb.impl.storage.ers.ram +package org.jacodb.util.collections +import org.jacodb.impl.storage.ers.ram.interned import java.util.Collections import java.util.NavigableMap import java.util.TreeMap diff --git a/jacodb-storage/src/test/kotlin/org/jacodb/impl/storage/ers/ram/SparseBitSetTest.kt b/jacodb-storage/src/test/kotlin/org/jacodb/util/collections/SparseBitSetTest.kt similarity index 98% rename from jacodb-storage/src/test/kotlin/org/jacodb/impl/storage/ers/ram/SparseBitSetTest.kt rename to jacodb-storage/src/test/kotlin/org/jacodb/util/collections/SparseBitSetTest.kt index f5f56bc97..af28bf632 100644 --- a/jacodb-storage/src/test/kotlin/org/jacodb/impl/storage/ers/ram/SparseBitSetTest.kt +++ b/jacodb-storage/src/test/kotlin/org/jacodb/util/collections/SparseBitSetTest.kt @@ -14,7 +14,7 @@ * limitations under the License. */ -package org.jacodb.impl.storage.ers.ram +package org.jacodb.util.collections import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue