Skip to content

Commit

Permalink
Fix unknown classes and ERS API improvement (#264)
Browse files Browse the repository at this point in the history
* Fix unknown classes and ERS API improvement

[jacodb-core] Specify identity for UnknownField, UnknownMethod, JcTypedFieldImpl

[jacodb-core] Fix operations with map names in LmdbKeyValueStorage

[jacodb-core] Avoid creation of named maps for R/O operations

[jacodb-core] Force use of sortedSet by getMapNames()

[jacodb-core] For an entity type, add ability to get names of used properties/blobs/links

The sets of names of properties/blobs/links returned by the API consist of names ever
have been set to an entity of specified type. So the sets can contain names of
properties/blobs/links that could actually have been deleted in all entities of specified type.

As a bonus, the way how named map are created was changed. As of now, any R/O operation
doesn't implicitly create name map, whereas R/W operation does.

TODO: fix implementation atop of LMDB and SQLite.

[jacodb-core] Define equals()/hashcode() in JcUnknownClass & JcUnknownType

[jacodb-core] Apply JcUnknownClassLookup only for instances of JcUnknownClass

As of now, there is no longer an ability to get and unknown field or
an unknown method for instances other than JcUnknownClass.

[jacodb-api] ERS API: inherit EntityIterable from Kotlin Sequence

As of now, EntityIterable is no longer Iterable, but Kotlin Sequence.
All methods and properties, except iterator(), have default implementations.
Binary operations are implemented as lazy sequences.

* [jacodb-core] Add to JcCacheSettings ability to specify cache SPI provider
  • Loading branch information
Saloed authored Aug 22, 2024
1 parent 549cc20 commit 788d041
Show file tree
Hide file tree
Showing 34 changed files with 622 additions and 253 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -16,29 +16,29 @@

package org.jacodb.api.jvm.storage.ers

interface EntityIterable : Iterable<Entity> {
interface EntityIterable : Sequence<Entity> {

val size: Long
val size: Long get() = toEntityIdSet().size.toLong()

val isEmpty: Boolean
val isEmpty: Boolean get() = size == 0L

val isNotEmpty: Boolean get() = !isEmpty

operator fun contains(e: Entity): Boolean
operator fun contains(e: Entity): Boolean = toEntityIdSet().contains(e.id)

operator fun plus(other: EntityIterable): EntityIterable = when (other) {
EMPTY -> this
else -> CollectionEntityIterable(union(other))
else -> this.union(other)
}

operator fun times(other: EntityIterable): EntityIterable = when (other) {
EMPTY -> EMPTY
else -> CollectionEntityIterable(intersect(other))
else -> this.intersect(other)
}

operator fun minus(other: EntityIterable): EntityIterable = when(other) {
operator fun minus(other: EntityIterable): EntityIterable = when (other) {
EMPTY -> this
else -> CollectionEntityIterable(subtract(other))
else -> this.subtract(other)
}

fun deleteAll() = forEach { entity -> entity.delete() }
Expand All @@ -49,8 +49,6 @@ interface EntityIterable : Iterable<Entity> {

override val size = 0L

override val isEmpty = true

override fun contains(e: Entity) = false

override fun iterator(): Iterator<Entity> = emptyList<Entity>().iterator()
Expand All @@ -64,15 +62,15 @@ interface EntityIterable : Iterable<Entity> {
}
}

class CollectionEntityIterable(private val set: Collection<Entity>) : EntityIterable {
class CollectionEntityIterable(private val c: Collection<Entity>) : EntityIterable {

override val size = set.size.toLong()
override val size = c.size.toLong()

override val isEmpty = set.isEmpty()
override val isEmpty = c.isEmpty()

override fun contains(e: Entity) = e in set
override fun contains(e: Entity) = e in c

override fun iterator() = set.iterator()
override fun iterator() = c.iterator()
}

class EntityIdCollectionEntityIterable(
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
/*
* Copyright 2022 UnitTestBot contributors (utbot.org)
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.jacodb.api.jvm.storage.ers

/**
* Default lazy implementations of binary operations over instances of `EntityIterable`
*/
fun EntityIterable.union(other: EntityIterable): EntityIterable {
val self = this
return object : EntityIterable {
override fun iterator(): Iterator<Entity> {
return object : Iterator<Entity> {

val iterated = HashSet<Entity>()
var iter = self.iterator()
var switchedToOther = false
var next: Entity? = null

override fun hasNext(): Boolean {
advance()
return next != null
}

override fun next(): Entity {
advance()
return (next ?: error("No more entities available")).also { next = null }
}

private fun advance() {
if (next == null) {
if (!switchedToOther) {
if (iter.hasNext()) {
next = iter.next().also {
iterated.add(it)
}
return
}
iter = other.iterator()
switchedToOther = true
}
while (iter.hasNext()) {
iter.next().let {
if (it !in iterated) {
next = it
return
}
}
}
}
}
}
}
}
}

fun EntityIterable.intersect(other: EntityIterable): EntityIterable {
val self = this
return object : EntityIterable {
override fun iterator(): Iterator<Entity> {
val otherSet = other.toEntityIdSet()
return self.filter { it.id in otherSet }.iterator()
}

}
}

fun EntityIterable.subtract(other: EntityIterable): EntityIterable {
val self = this
return object : EntityIterable {
override fun iterator(): Iterator<Entity> {
val otherSet = other.toEntityIdSet()
return self.filter { it.id !in otherSet }.iterator()
}

}
}

fun EntityIterable.toEntityIdSet(): Set<EntityId> {
val it = iterator()
if (!it.hasNext()) {
return emptySet()
}
val element = it.next()
if (!it.hasNext()) {
return setOf(element.id)
}
val result = LinkedHashSet<EntityId>()
result.add(element.id)
while (it.hasNext()) {
result.add(it.next().id)
}
return result
}
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,21 @@ interface Transaction : Closeable {
*/
fun getTypeId(type: String): Int

/**
* Returns set of property names ever being set to an entity of specified `type`.
*/
fun getPropertyNames(type: String): Set<String> = emptySet()

/**
* Returns set of blob names ever being set to an entity of specified `type`.
*/
fun getBlobNamesNames(type: String): Set<String> = emptySet()

/**
* Returns set of link names ever being set to an entity of specified `type`.
*/
fun getLinkNamesNames(type: String): Set<String> = emptySet()

fun all(type: String): EntityIterable

fun <T : Any> find(type: String, propertyName: String, value: T): EntityIterable
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ abstract class PluggableKeyValueStorage : Closeable {

fun delete(map: String, key: ByteArray) = transactional { txn -> txn.delete(map, key) }

fun mapSize(map: String): Long = transactional { txn -> txn.getNamedMap(map).size(txn) }
fun mapSize(map: String): Long = transactional { txn -> txn.getNamedMap(map, create = false)?.size(txn) } ?: 0L

fun all(map: String): List<Pair<ByteArray, ByteArray>> = readonlyTransactional { txn ->
buildList {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,25 +26,29 @@ interface Transaction : Closeable {

val isFinished: Boolean

fun getNamedMap(name: String): NamedMap
fun getNamedMap(name: String, create: Boolean = false): NamedMap?

fun get(map: String, key: ByteArray): ByteArray? = get(getNamedMap(map), key)
fun getMapNames(): Set<String>

fun get(map: String, key: ByteArray): ByteArray? = getNamedMap(map, create = false)?.let { get(it, key) }

fun get(map: NamedMap, key: ByteArray): ByteArray?

fun put(map: String, key: ByteArray, value: ByteArray): Boolean = put(getNamedMap(map), key, value)
fun put(map: String, key: ByteArray, value: ByteArray): Boolean = put(getNamedMap(map, create = true)!!, key, value)

fun put(map: NamedMap, key: ByteArray, value: ByteArray): Boolean

fun delete(map: String, key: ByteArray): Boolean = delete(getNamedMap(map), key)
fun delete(map: String, key: ByteArray): Boolean = getNamedMap(map, create = false)?.let { delete(it, key) } == true

fun delete(map: NamedMap, key: ByteArray): Boolean

fun delete(map: String, key: ByteArray, value: ByteArray): Boolean = delete(getNamedMap(map), key, value)
fun delete(map: String, key: ByteArray, value: ByteArray): Boolean =
getNamedMap(map, create = false)?.let { delete(it, key, value) } == true

fun delete(map: NamedMap, key: ByteArray, value: ByteArray): Boolean

fun navigateTo(map: String, key: ByteArray? = null): Cursor = navigateTo(getNamedMap(map), key)
fun navigateTo(map: String, key: ByteArray? = null): Cursor =
getNamedMap(map, create = false)?.let { navigateTo(it, key) } ?: EmptyCursor(this)

fun navigateTo(map: NamedMap, key: ByteArray? = null): Cursor

Expand All @@ -64,13 +68,6 @@ interface NamedMap {
fun size(txn: Transaction): Long
}

object EmptyNamedMap : NamedMap {

override val name = "EmptyNamedMap"

override fun size(txn: Transaction) = 0L
}

abstract class TransactionDecorator(val decorated: Transaction) : Transaction by decorated

fun Transaction.withFinishedState(): Transaction = WithFinishedCheckingTxn(this)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -93,11 +93,11 @@ object Approximations : JcFeature<Any?, Any?>, JcClassExtFeature, JcInstExtFeatu
},
noSqlAction = { txn ->
val valueId = persistence.findSymbolId("value")
txn.find("Annotation", "nameId", approxSymbol.compressed).asSequence()
txn.find("Annotation", "nameId", approxSymbol.compressed)
.filter { it.getCompressedBlob<Int>("refKind") == RefKind.CLASS.ordinal }
.flatMap { annotation ->
annotation.getLink("ref").let { clazz ->
annotation.getLinks("values").asSequence().map { clazz to it }
annotation.getLinks("values").map { clazz to it }
}
}.filter { (_, annotationValue) ->
valueId == annotationValue["nameId"]
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ class JCDBSymbolsInternerImpl : JCDBSymbolsInterner, Closeable {
val unwrapped = txn.unwrap
if (unwrapped is KVErsTransaction) {
val kvTxn = unwrapped.kvTxn
val symbolsMap = kvTxn.getNamedMap(symbolsMapName)
val symbolsMap = kvTxn.getNamedMap(symbolsMapName, create = true)!!
val stringBinding = BuiltInBindingProvider.getBinding(String::class.java)
val longBinding = BuiltInBindingProvider.getBinding(Long::class.java)
entries.forEach { (name, id) ->
Expand Down
36 changes: 25 additions & 11 deletions jacodb-core/src/main/kotlin/org/jacodb/impl/JcClasspathImpl.kt
Original file line number Diff line number Diff line change
Expand Up @@ -16,10 +16,27 @@

package org.jacodb.impl

import kotlinx.coroutines.*
import org.jacodb.api.jvm.*
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.async
import kotlinx.coroutines.isActive
import kotlinx.coroutines.joinAll
import kotlinx.coroutines.withContext
import org.jacodb.api.jvm.ClassSource
import org.jacodb.api.jvm.JcAnnotation
import org.jacodb.api.jvm.JcArrayType
import org.jacodb.api.jvm.JcByteCodeLocation
import org.jacodb.api.jvm.JcClassOrInterface
import org.jacodb.api.jvm.JcClasspath
import org.jacodb.api.jvm.JcClasspathExtFeature
import org.jacodb.api.jvm.JcClasspathExtFeature.JcResolvedClassResult
import org.jacodb.api.jvm.JcClasspathExtFeature.JcResolvedTypeResult
import org.jacodb.api.jvm.JcClasspathFeature
import org.jacodb.api.jvm.JcClasspathTask
import org.jacodb.api.jvm.JcFeatureEvent
import org.jacodb.api.jvm.JcRefType
import org.jacodb.api.jvm.JcType
import org.jacodb.api.jvm.PredefinedPrimitives
import org.jacodb.api.jvm.RegisteredLocation
import org.jacodb.api.jvm.ext.JAVA_OBJECT
import org.jacodb.api.jvm.ext.toType
import org.jacodb.impl.bytecode.JcClassOrInterfaceImpl
Expand Down Expand Up @@ -50,15 +67,12 @@ class JcClasspathImpl(
override val registeredLocations: List<RegisteredLocation> = locationsRegistrySnapshot.locations
override val registeredLocationIds: Set<Long> = locationsRegistrySnapshot.ids
private val classpathVfs = ClasspathVfs(globalClassVFS, locationsRegistrySnapshot)
private val featuresChain = run {
val strictFeatures = features.filter { it !is UnknownClasses }
val hasUnknownClasses = strictFeatures.size != features.size
JcFeaturesChain(
strictFeatures + listOfNotNull(
JcClasspathFeatureImpl(),
UnknownClasses.takeIf { hasUnknownClasses })
)
}
private val featuresChain = JcFeaturesChain(
if (!features.any { it is UnknownClasses }) {
features + JcClasspathFeatureImpl()
} else {
features.filter { it !is UnknownClasses } + JcClasspathFeatureImpl() + UnknownClasses
})

override suspend fun refreshed(closeOld: Boolean): JcClasspath {
return db.new(this).also {
Expand Down
4 changes: 2 additions & 2 deletions jacodb-core/src/main/kotlin/org/jacodb/impl/JcSettings.kt
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import org.jacodb.api.jvm.JcPersistenceImplSettings
import org.jacodb.api.jvm.JcPersistenceSettings
import org.jacodb.api.jvm.storage.ers.EmptyErsSettings
import org.jacodb.api.jvm.storage.ers.ErsSettings
import org.jacodb.impl.caches.guava.GUAVA_CACHE_PROVIDER_ID
import org.jacodb.impl.storage.SQLITE_DATABASE_PERSISTENCE_SPI
import org.jacodb.impl.storage.ers.ERS_DATABASE_PERSISTENCE_SPI
import org.jacodb.impl.storage.ers.kv.KV_ERS_SPI
Expand Down Expand Up @@ -187,8 +188,8 @@ data class JcCacheSegmentSettings(

enum class ValueStoreType { WEAK, SOFT, STRONG }


class JcCacheSettings {
var cacheSpiId: String = GUAVA_CACHE_PROVIDER_ID
var classes: JcCacheSegmentSettings = JcCacheSegmentSettings()
var types: JcCacheSegmentSettings = JcCacheSegmentSettings()
var rawInstLists: JcCacheSegmentSettings = JcCacheSegmentSettings()
Expand Down Expand Up @@ -223,7 +224,6 @@ class JcCacheSettings {
flowGraphs =
JcCacheSegmentSettings(maxSize = maxSize, expiration = expiration, valueStoreType = valueStoreType)
}

}

object JcSQLitePersistenceSettings : JcPersistenceImplSettings {
Expand Down
Loading

0 comments on commit 788d041

Please sign in to comment.