Fix unknown classes and ERS API improvement

jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/storage/ers/EntityIterable.kt

@@ -16,29 +16,29 @@
 
 package org.jacodb.api.jvm.storage.ers
 
-interface EntityIterable : Iterable<Entity> {
+interface EntityIterable : Sequence<Entity> {
 
-    val size: Long
+    val size: Long get() = toEntityIdSet().size.toLong()
 
-    val isEmpty: Boolean
+    val isEmpty: Boolean get() = size == 0L
 
     val isNotEmpty: Boolean get() = !isEmpty
 
-    operator fun contains(e: Entity): Boolean
+    operator fun contains(e: Entity): Boolean = toEntityIdSet().contains(e.id)
 
     operator fun plus(other: EntityIterable): EntityIterable = when (other) {
         EMPTY -> this
-        else -> CollectionEntityIterable(union(other))
+        else -> this.union(other)
     }
 
     operator fun times(other: EntityIterable): EntityIterable = when (other) {
         EMPTY -> EMPTY
-        else -> CollectionEntityIterable(intersect(other))
+        else -> this.intersect(other)
     }
 
-    operator fun minus(other: EntityIterable): EntityIterable = when(other) {
+    operator fun minus(other: EntityIterable): EntityIterable = when (other) {
         EMPTY -> this
-        else -> CollectionEntityIterable(subtract(other))
+        else -> this.subtract(other)
     }
 
     fun deleteAll() = forEach { entity -> entity.delete() }
@@ -49,8 +49,6 @@ interface EntityIterable : Iterable<Entity> {
 
             override val size = 0L
 
-            override val isEmpty = true
-
             override fun contains(e: Entity) = false
 
             override fun iterator(): Iterator<Entity> = emptyList<Entity>().iterator()
@@ -64,15 +62,15 @@ interface EntityIterable : Iterable<Entity> {
     }
 }
 
-class CollectionEntityIterable(private val set: Collection<Entity>) : EntityIterable {
+class CollectionEntityIterable(private val c: Collection<Entity>) : EntityIterable {
 
-    override val size = set.size.toLong()
+    override val size = c.size.toLong()
 
-    override val isEmpty = set.isEmpty()
+    override val isEmpty = c.isEmpty()
 
-    override fun contains(e: Entity) = e in set
+    override fun contains(e: Entity) = e in c
 
-    override fun iterator() = set.iterator()
+    override fun iterator() = c.iterator()
 }
 
 class EntityIdCollectionEntityIterable(

jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/storage/ers/EntityIterableEx.kt

@@ -0,0 +1,107 @@
+/*
+ *  Copyright 2022 UnitTestBot contributors (utbot.org)
+ * <p>
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ * <p>
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.jacodb.api.jvm.storage.ers
+
+/**
+ * Default lazy implementations of binary operations over instances of `EntityIterable`
+ */
+fun EntityIterable.union(other: EntityIterable): EntityIterable {
+    val self = this
+    return object : EntityIterable {
+        override fun iterator(): Iterator<Entity> {
+            return object : Iterator<Entity> {
+
+                val iterated = HashSet<Entity>()
+                var iter = self.iterator()
+                var switchedToOther = false
+                var next: Entity? = null
+
+                override fun hasNext(): Boolean {
+                    advance()
+                    return next != null
+                }
+
+                override fun next(): Entity {
+                    advance()
+                    return (next ?: error("No more entities available")).also { next = null }
+                }
+
+                private fun advance() {
+                    if (next == null) {
+                        if (!switchedToOther) {
+                            if (iter.hasNext()) {
+                                next = iter.next().also {
+                                    iterated.add(it)
+                                }
+                                return
+                            }
+                            iter = other.iterator()
+                            switchedToOther = true
+                        }
+                        while (iter.hasNext()) {
+                            iter.next().let {
+                                if (it !in iterated) {
+                                    next = it
+                                    return
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
+    }
+}
+
+fun EntityIterable.intersect(other: EntityIterable): EntityIterable {
+    val self = this
+    return object : EntityIterable {
+        override fun iterator(): Iterator<Entity> {
+            val otherSet = other.toEntityIdSet()
+            return self.filter { it.id in otherSet }.iterator()
+        }
+
+    }
+}
+
+fun EntityIterable.subtract(other: EntityIterable): EntityIterable {
+    val self = this
+    return object : EntityIterable {
+        override fun iterator(): Iterator<Entity> {
+            val otherSet = other.toEntityIdSet()
+            return self.filter { it.id !in otherSet }.iterator()
+        }
+
+    }
+}
+
+fun EntityIterable.toEntityIdSet(): Set<EntityId> {
+    val it = iterator()
+    if (!it.hasNext()) {
+        return emptySet()
+    }
+    val element = it.next()
+    if (!it.hasNext()) {
+        return setOf(element.id)
+    }
+    val result = LinkedHashSet<EntityId>()
+    result.add(element.id)
+    while (it.hasNext()) {
+        result.add(it.next().id)
+    }
+    return result
+}

jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/storage/ers/Transaction.kt

@@ -45,6 +45,21 @@ interface Transaction : Closeable {
      */
     fun getTypeId(type: String): Int
 
+    /**
+     * Returns set of property names ever being set to an entity of specified `type`.
+     */
+    fun getPropertyNames(type: String): Set<String> = emptySet()
+
+    /**
+     * Returns set of blob names ever being set to an entity of specified `type`.
+     */
+    fun getBlobNamesNames(type: String): Set<String> = emptySet()
+
+    /**
+     * Returns set of link names ever being set to an entity of specified `type`.
+     */
+    fun getLinkNamesNames(type: String): Set<String> = emptySet()
+
     fun all(type: String): EntityIterable
 
     fun <T : Any> find(type: String, propertyName: String, value: T): EntityIterable

jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/storage/kv/PluggableKeyValueStorage.kt

@@ -38,7 +38,7 @@ abstract class PluggableKeyValueStorage : Closeable {
 
     fun delete(map: String, key: ByteArray) = transactional { txn -> txn.delete(map, key) }
 
-    fun mapSize(map: String): Long = transactional { txn -> txn.getNamedMap(map).size(txn) }
+    fun mapSize(map: String): Long = transactional { txn -> txn.getNamedMap(map, create = false)?.size(txn) } ?: 0L
 
     fun all(map: String): List<Pair<ByteArray, ByteArray>> = readonlyTransactional { txn ->
         buildList {

jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/storage/kv/Transaction.kt

@@ -26,25 +26,29 @@ interface Transaction : Closeable {
 
     val isFinished: Boolean
 
-    fun getNamedMap(name: String): NamedMap
+    fun getNamedMap(name: String, create: Boolean = false): NamedMap?
 
-    fun get(map: String, key: ByteArray): ByteArray? = get(getNamedMap(map), key)
+    fun getMapNames(): Set<String>
+
+    fun get(map: String, key: ByteArray): ByteArray? = getNamedMap(map, create = false)?.let { get(it, key) }
 
     fun get(map: NamedMap, key: ByteArray): ByteArray?
 
-    fun put(map: String, key: ByteArray, value: ByteArray): Boolean = put(getNamedMap(map), key, value)
+    fun put(map: String, key: ByteArray, value: ByteArray): Boolean = put(getNamedMap(map, create = true)!!, key, value)
 
     fun put(map: NamedMap, key: ByteArray, value: ByteArray): Boolean
 
-    fun delete(map: String, key: ByteArray): Boolean = delete(getNamedMap(map), key)
+    fun delete(map: String, key: ByteArray): Boolean = getNamedMap(map, create = false)?.let { delete(it, key) } == true
 
     fun delete(map: NamedMap, key: ByteArray): Boolean
 
-    fun delete(map: String, key: ByteArray, value: ByteArray): Boolean = delete(getNamedMap(map), key, value)
+    fun delete(map: String, key: ByteArray, value: ByteArray): Boolean =
+        getNamedMap(map, create = false)?.let { delete(it, key, value) } == true
 
     fun delete(map: NamedMap, key: ByteArray, value: ByteArray): Boolean
 
-    fun navigateTo(map: String, key: ByteArray? = null): Cursor = navigateTo(getNamedMap(map), key)
+    fun navigateTo(map: String, key: ByteArray? = null): Cursor =
+        getNamedMap(map, create = false)?.let { navigateTo(it, key) } ?: EmptyCursor(this)
 
     fun navigateTo(map: NamedMap, key: ByteArray? = null): Cursor
 
@@ -64,13 +68,6 @@ interface NamedMap {
     fun size(txn: Transaction): Long
 }
 
-object EmptyNamedMap : NamedMap {
-
-    override val name = "EmptyNamedMap"
-
-    override fun size(txn: Transaction) = 0L
-}
-
 abstract class TransactionDecorator(val decorated: Transaction) : Transaction by decorated
 
 fun Transaction.withFinishedState(): Transaction = WithFinishedCheckingTxn(this)

jacodb-approximations/src/main/kotlin/org/jacodb/approximation/Approximations.kt

@@ -93,11 +93,11 @@
                     },
                     noSqlAction = { txn ->
                         val valueId = persistence.findSymbolId("value")
-                        txn.find("Annotation", "nameId", approxSymbol.compressed).asSequence()
+                        txn.find("Annotation", "nameId", approxSymbol.compressed)
                             .filter { it.getCompressedBlob<Int>("refKind") == RefKind.CLASS.ordinal }
                             .flatMap { annotation ->
                                 annotation.getLink("ref").let { clazz ->
-                                    annotation.getLinks("values").asSequence().map { clazz to it }
+                                    annotation.getLinks("values").map { clazz to it }
                                 }
                             }.filter { (_, annotationValue) ->
                                 valueId == annotationValue["nameId"]

jacodb-core/src/main/kotlin/org/jacodb/impl/JCDBSymbolsInternerImpl.kt

@@ -121,7 +121,7 @@
                     val unwrapped = txn.unwrap
                     if (unwrapped is KVErsTransaction) {
                         val kvTxn = unwrapped.kvTxn
-                        val symbolsMap = kvTxn.getNamedMap(symbolsMapName)
+                        val symbolsMap = kvTxn.getNamedMap(symbolsMapName, create = true)!!
                         val stringBinding = BuiltInBindingProvider.getBinding(String::class.java)
                         val longBinding = BuiltInBindingProvider.getBinding(Long::class.java)
                         entries.forEach { (name, id) ->

jacodb-core/src/main/kotlin/org/jacodb/impl/JcClasspathImpl.kt

@@ -16,10 +16,27 @@
 
 package org.jacodb.impl
 
-import kotlinx.coroutines.*
-import org.jacodb.api.jvm.*
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.async
+import kotlinx.coroutines.isActive
+import kotlinx.coroutines.joinAll
+import kotlinx.coroutines.withContext
+import org.jacodb.api.jvm.ClassSource
+import org.jacodb.api.jvm.JcAnnotation
+import org.jacodb.api.jvm.JcArrayType
+import org.jacodb.api.jvm.JcByteCodeLocation
+import org.jacodb.api.jvm.JcClassOrInterface
+import org.jacodb.api.jvm.JcClasspath
+import org.jacodb.api.jvm.JcClasspathExtFeature
 import org.jacodb.api.jvm.JcClasspathExtFeature.JcResolvedClassResult
 import org.jacodb.api.jvm.JcClasspathExtFeature.JcResolvedTypeResult
+import org.jacodb.api.jvm.JcClasspathFeature
+import org.jacodb.api.jvm.JcClasspathTask
+import org.jacodb.api.jvm.JcFeatureEvent
+import org.jacodb.api.jvm.JcRefType
+import org.jacodb.api.jvm.JcType
+import org.jacodb.api.jvm.PredefinedPrimitives
+import org.jacodb.api.jvm.RegisteredLocation
 import org.jacodb.api.jvm.ext.JAVA_OBJECT
 import org.jacodb.api.jvm.ext.toType
 import org.jacodb.impl.bytecode.JcClassOrInterfaceImpl
@@ -50,15 +67,12 @@ class JcClasspathImpl(
     override val registeredLocations: List<RegisteredLocation> = locationsRegistrySnapshot.locations
     override val registeredLocationIds: Set<Long> = locationsRegistrySnapshot.ids
     private val classpathVfs = ClasspathVfs(globalClassVFS, locationsRegistrySnapshot)
-    private val featuresChain = run {
-        val strictFeatures = features.filter { it !is UnknownClasses }
-        val hasUnknownClasses = strictFeatures.size != features.size
-        JcFeaturesChain(
-            strictFeatures + listOfNotNull(
-                JcClasspathFeatureImpl(),
-                UnknownClasses.takeIf { hasUnknownClasses })
-        )
-    }
+    private val featuresChain = JcFeaturesChain(
+        if (!features.any { it is UnknownClasses }) {
+            features + JcClasspathFeatureImpl()
+        } else {
+            features.filter { it !is UnknownClasses } + JcClasspathFeatureImpl() + UnknownClasses
+        })
 
     override suspend fun refreshed(closeOld: Boolean): JcClasspath {
         return db.new(this).also {

jacodb-core/src/main/kotlin/org/jacodb/impl/JcSettings.kt

@@ -25,6 +25,7 @@ import org.jacodb.api.jvm.JcPersistenceImplSettings
 import org.jacodb.api.jvm.JcPersistenceSettings
 import org.jacodb.api.jvm.storage.ers.EmptyErsSettings
 import org.jacodb.api.jvm.storage.ers.ErsSettings
+import org.jacodb.impl.caches.guava.GUAVA_CACHE_PROVIDER_ID
 import org.jacodb.impl.storage.SQLITE_DATABASE_PERSISTENCE_SPI
 import org.jacodb.impl.storage.ers.ERS_DATABASE_PERSISTENCE_SPI
 import org.jacodb.impl.storage.ers.kv.KV_ERS_SPI
@@ -187,8 +188,8 @@ data class JcCacheSegmentSettings(
 
 enum class ValueStoreType { WEAK, SOFT, STRONG }
 
-
 class JcCacheSettings {
+    var cacheSpiId: String = GUAVA_CACHE_PROVIDER_ID
     var classes: JcCacheSegmentSettings = JcCacheSegmentSettings()
     var types: JcCacheSegmentSettings = JcCacheSegmentSettings()
     var rawInstLists: JcCacheSegmentSettings = JcCacheSegmentSettings()
@@ -223,7 +224,6 @@ class JcCacheSettings {
             flowGraphs =
                 JcCacheSegmentSettings(maxSize = maxSize, expiration = expiration, valueStoreType = valueStoreType)
         }
-
 }
 
 object JcSQLitePersistenceSettings : JcPersistenceImplSettings {