ERS Fixes

[jacodb-core] Make RAM benchmarks depending on the downloadAndUnzipIdeaCommunity task

[jacodb-core] In RegisteredLocation.lazySources, never return sources with loaded bytecode

[jacodb-api] Add typed ERS API for range queries

[jacodb-core] On closing persistence, close symbol interner, don't setup

[jacodb-core] Fix HierarchyExtensionERS to not returning EntityIterable accessible outside a transaction

[jacodb-core] Fix JCDBSymbolsInternerImpl.setup() atop of KVEntityRelationshipStorage

[jacodb-core] In ErsPersistenceImpl.read(), use optimistic transactions only for RAMEntityRelationshipStorage

[jacodb-core] Fix HierarchyExtension working with ERS API persistence without installed InMemoryHierarchy feature

The "inherits" relation is being saved using property name.
The "implements" relation is being saved using two-way entity links with dedicate entity of type "Interface".

[jacodb-core] Avoid duplicates in output of HierarchyExtension.findOverrides() implemented with ERS API

[jacodb-core] Add RestoredXodusDBTest

[jacodb-core] Fix and optimize JcClasspath.allClassesExceptObject() in noSqlAction

[jacodb-core] Fix InMemoryHierarchy's BeforeIndexing noSqlAction

jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/storage/ers/readme.md

@@ -293,10 +293,10 @@ There are the following basic queries available on the instance of `Transaction`
 - `Transaction.all()` gets all entities of specified type;
 - `Transaction.find()` gets entities of specified type with specified property equal to specified value;
 - `Transaction.findLt()` gets entities of specified type with specified property less than specified value;
-- `Transaction.findOrLt()` gets entities of specified type with specified property equal to or less than specified
+- `Transaction.findEqOrLt()` gets entities of specified type with specified property equal to or less than specified
   value;
 - `Transaction.findGt()` gets entities of specified type with specified property greater than specified value;
-- `Transaction.findOrGt()` gets entities of specified type with specified property equal to or greater than specified
+- `Transaction.findEqOrGt()` gets entities of specified type with specified property equal to or greater than specified
   value.
 
 Enumerate all users:

jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/storage/ers/typed/TypedErsApi.kt

@@ -19,6 +19,7 @@ package org.jacodb.api.jvm.storage.ers.typed
 import org.jacodb.api.jvm.storage.ers.Entity
 import org.jacodb.api.jvm.storage.ers.EntityId
 import org.jacodb.api.jvm.storage.ers.EntityIterable
+import org.jacodb.api.jvm.storage.ers.FindOption
 import org.jacodb.api.jvm.storage.ers.Transaction
 
 fun <ENTITY_TYPE : ErsType> Transaction.newEntity(type: ENTITY_TYPE): TypedEntity<ENTITY_TYPE> =
@@ -30,6 +31,8 @@ fun <ENTITY_TYPE : ErsType> Transaction.getEntityOrNull(id: TypedEntityId<ENTITY
 fun <ENTITY_TYPE : ErsType> Transaction.getTypeId(type: ENTITY_TYPE): TypeId<ENTITY_TYPE> =
     TypeIdImpl(getTypeId(type.typeName))
 
+/// region find extension functions
+
 fun <ENTITY_TYPE : ErsType, VALUE : Any> Transaction.find(
     property: ErsProperty<ENTITY_TYPE, VALUE, ErsSearchability.Searchable>,
     value: VALUE
@@ -41,6 +44,65 @@ fun <ENTITY_TYPE : ErsType, VALUE : Any> Transaction.find(
     )
 )
 
+fun <ENTITY_TYPE : ErsType, VALUE : Any> Transaction.findLt(
+    property: ErsProperty<ENTITY_TYPE, VALUE, ErsSearchability.Searchable>,
+    value: VALUE
+): TypedEntityIterable<ENTITY_TYPE> = TypedEntityIterableImpl(
+    findLt(
+        type = property.ownerType.typeName,
+        propertyName = property.name,
+        value = value
+    )
+)
+
+fun <ENTITY_TYPE : ErsType, VALUE : Any> Transaction.findEqOrLt(
+    property: ErsProperty<ENTITY_TYPE, VALUE, ErsSearchability.Searchable>,
+    value: VALUE
+): TypedEntityIterable<ENTITY_TYPE> = TypedEntityIterableImpl(
+    findEqOrLt(
+        type = property.ownerType.typeName,
+        propertyName = property.name,
+        value = value
+    )
+)
+
+fun <ENTITY_TYPE : ErsType, VALUE : Any> Transaction.findGt(
+    property: ErsProperty<ENTITY_TYPE, VALUE, ErsSearchability.Searchable>,
+    value: VALUE
+): TypedEntityIterable<ENTITY_TYPE> = TypedEntityIterableImpl(
+    findGt(
+        type = property.ownerType.typeName,
+        propertyName = property.name,
+        value = value
+    )
+)
+
+fun <ENTITY_TYPE : ErsType, VALUE : Any> Transaction.findEqOrGt(
+    property: ErsProperty<ENTITY_TYPE, VALUE, ErsSearchability.Searchable>,
+    value: VALUE
+): TypedEntityIterable<ENTITY_TYPE> = TypedEntityIterableImpl(
+    findEqOrGt(
+        type = property.ownerType.typeName,
+        propertyName = property.name,
+        value = value
+    )
+)
+
+fun <ENTITY_TYPE : ErsType, VALUE : Any> Transaction.find(
+    property: ErsProperty<ENTITY_TYPE, VALUE, ErsSearchability.Searchable>,
+    value: VALUE,
+    option: FindOption
+): TypedEntityIterable<ENTITY_TYPE> = TypedEntityIterableImpl(
+    find(
+        type = property.ownerType.typeName,
+        propertyName = property.name,
+        value = value,
+        option = option
+    )
+)
+
+/// endregion
+
 fun <ENTITY_TYPE : ErsType> Transaction.all(type: ENTITY_TYPE): TypedEntityIterable<ENTITY_TYPE> =
     TypedEntityIterableImpl(all(type.typeName))
 

jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/storage/ers/typed/readme.md

@@ -26,6 +26,7 @@ Inside the entity type, you can declare which properties and links it has:
 object UserType : ErsType {
     val login by property(String::class)
     val password by property(String::class)
+    val age by property(Int::class)
     val avatar by property(ByteArray::class, searchability = ErsSearchability.NonSearchable)
     val profile by link(UserProfileType)
 }
@@ -70,7 +71,18 @@ Next, you can:
 
 Currently, there are two basic queries available on the instance of `Transaction`:
 
+There are the following basic queries available on the instance of `Transaction`:
 
+- `Transaction.all()` gets all entities of specified type;
+- `Transaction.find()` gets entities with specified property equal to specified value;
+- `Transaction.findLt()` gets entities with specified property less than specified value;
+- `Transaction.findEqOrLt()` gets entities with specified property equal to or less than specified
+  value;
+- `Transaction.findGt()` gets entities with specified property greater than specified value;
+- `Transaction.findEqOrGt()` gets entities with specified property equal to or greater than specified
+  value.
+
+Simple examples:
 1. Enumerate all users:
     ```kotlin
     txn.all(UserType).forEach { user ->
@@ -83,3 +95,29 @@ Currently, there are two basic queries available on the instance of `Transaction
         // ...
     }
     ```
+3. Enumerate all users having age equal to or less than 42:
+   ```kotlin
+   txn.findEqOrLt(UserType.age, 42).forEach { user ->
+       // ...
+   }
+   ```
+
+Queries return instances of `TypedEntityIterable<ENTITY_TYPE>`. `TypedEntityIterable<ENTITY_TYPE>` is 
+`Iterable<TypedEntity<ENTITY_TYPE>>`. In addition, there are tree binary operations on instances of `EntityIterable`:
+_intersect_, _union_ (`+`) and _minus_ (`-`).
+
+**NOTE:** some ERS implementations can use two consecutive queries to implement these operations.
+
+Query combination examples:
+1. To get users with age `42` _and_ having login `[email protected]`:
+    ```kotlin
+    val users = txn.find(UserType.age, 42).intersect(txn.find(UserType.login, "[email protected]")) 
+    ```
+2. To get users with age equal to or greater than `42` _or_ having login `[email protected]`:
+    ```kotlin
+    val users = txn.findEqOrGt(UserType.age, 42) + txn.find(UserType.login, "[email protected]") 
+    ```
+3. To get users with age greater than `42` _not_ having login `[email protected]`:
+    ```kotlin
+    val users = txn.findGt(UserType.age, 42) - txn.find(UserType.login, "[email protected]") 
+    ```

jacodb-benchmarks/build.gradle.kts

@@ -96,8 +96,10 @@ tasks.register<Copy>("downloadAndUnzipIdeaCommunity") {
 
 val benchmarkTasks = listOf(
     "testJcdbBenchmark",
+    "testJcdbRAMBenchmark",
     "testSootBenchmark",
     "testAwaitBackgroundBenchmark",
+    "testRamAwaitBackgroundBenchmark"
 )
 tasks.matching { it.name in benchmarkTasks }.configureEach {
     dependsOn("downloadAndUnzipIdeaCommunity")

jacodb-core/src/main/kotlin/org/jacodb/impl/JCDBSymbolsInternerImpl.kt

@@ -69,7 +69,7 @@ class JCDBSymbolsInternerImpl : JCDBSymbolsInterner, Closeable {
                     val stringBinding = BuiltInBindingProvider.getBinding(String::class.java)
                     val longBinding = BuiltInBindingProvider.getBinding(Long::class.java)
                     kvTxn.navigateTo(symbolsMapName).forEach { idBytes, nameBytes ->
-                        val id = longBinding.getObject(idBytes)
+                        val id = longBinding.getObjectCompressed(idBytes)
                         val name = stringBinding.getObject(nameBytes)
                         symbolsCache[name] = id
                         idCache[id] = name

jacodb-core/src/main/kotlin/org/jacodb/impl/features/HierarchyExtension.kt

@@ -33,7 +33,6 @@ import org.jacodb.api.jvm.storage.ers.Entity
 import org.jacodb.api.jvm.storage.ers.EntityIterable
 import org.jacodb.api.jvm.storage.ers.Transaction
 import org.jacodb.api.jvm.storage.ers.compressed
-import org.jacodb.api.jvm.storage.ers.links
 import org.jacodb.impl.asSymbolId
 import org.jacodb.impl.fs.PersistenceClassSource
 import org.jacodb.impl.storage.BatchedSequence
@@ -97,9 +96,9 @@ internal fun JcClasspath.allClassesExceptObject(context: JCDBContext, direct: Bo
         noSqlAction = { txn ->
             val objectNameId = db.persistence.findSymbolId(JAVA_OBJECT)
             txn.all("Class").asSequence().filter { clazz ->
-                clazz.getCompressed<Long>("nameId") != objectNameId &&
+                (!direct || clazz.getCompressed<Long>("inherits") == null) &&
                         clazz.getCompressed<Long>("locationId") in locationIds &&
-                        (!direct || links(clazz, "inherits").asIterable.isEmpty)
+                        clazz.getCompressed<Long>("nameId") != objectNameId
             }.toClassSourceSequence(db).toList().asSequence()
         }
     )
@@ -164,16 +163,16 @@ private class HierarchyExtensionERS(cp: JcClasspath) : HierarchyExtensionBase(cp
                 val locationIds = cp.registeredLocations.mapTo(mutableSetOf()) { it.id }
                 val nameId = name.asSymbolId(persistence.symbolInterner)
                 if (entireHierarchy) {
-                    entireHierarchy(txn, nameId, mutableListOf())
+                    entireHierarchy(txn, nameId, mutableSetOf())
                 } else {
                     directSubClasses(txn, nameId)
                 }.asSequence().filter { clazz -> clazz.getCompressed<Long>("locationId") in locationIds }
                     .toClassSourceSequence(db)
-            }
-        }.map { cp.toJcClass(it) }
+            }.map { cp.toJcClass(it) }.toList().asSequence()
+        }
     }
 
-    private fun entireHierarchy(txn: Transaction, nameId: Long, result: MutableList<Entity>): Iterable<Entity> {
+    private fun entireHierarchy(txn: Transaction, nameId: Long, result: MutableSet<Entity>): Iterable<Entity> {
         val subClasses = directSubClasses(txn, nameId)
         if (subClasses.isNotEmpty) {
             result += subClasses
@@ -185,8 +184,11 @@ private class HierarchyExtensionERS(cp: JcClasspath) : HierarchyExtensionBase(cp
     }
 
     private fun directSubClasses(txn: Transaction, nameId: Long): EntityIterable {
-        val clazz = txn.find("Class", "nameId", nameId.compressed).firstOrNull() ?: return EntityIterable.EMPTY
-        return (links(clazz, "inheritedBy").asIterable + links(clazz, "implementedBy").asIterable)
+        val nameIdCompressed = nameId.compressed
+        txn.find("Interface", "nameId", nameIdCompressed).firstOrNull()?.let { i ->
+            return i.getLinks("implementedBy")
+        }
+        return txn.find("Class", "inherits", nameIdCompressed)
     }
 }
 

jacodb-core/src/main/kotlin/org/jacodb/impl/features/InMemoryHierarchy.kt

@@ -105,10 +105,12 @@ object InMemoryHierarchy : JcFeature<InMemoryHierarchyReq, ClassSource> {
                             txn.all("Class").map { clazz ->
                                 val locationId: Long? = clazz.getCompressed("locationId")
                                 val classSymbolId: Long? = clazz.getCompressed("nameId")
-                                val superClasses =
-                                    links(clazz, "inherits").asIterable + links(clazz, "implements").asIterable
-                                superClasses.forEach { superClass ->
-                                    val nameId by propertyOf<Long>(superClass, compressed = true)
+                                val superClasses = mutableListOf<Long>()
+                                clazz.getCompressed<Long>("inherits")?.let { nameId -> superClasses += nameId }
+                                links(clazz, "implements").asIterable.forEach { anInterface ->
+                                    anInterface.getCompressed<Long>("nameId")?.let { nameId -> superClasses += nameId }
+                                }
+                                superClasses.forEach { nameId ->
                                     result += (Triple(classSymbolId, nameId, locationId))
                                 }
                             }

jacodb-core/src/main/kotlin/org/jacodb/impl/fs/ByteCodeLoaderImpl.kt

@@ -28,11 +28,7 @@ val RegisteredLocation.sources: List<ClassSource>
 
 val RegisteredLocation.lazySources: List<ClassSource>
     get() {
-        val classNames = jcLocation?.classNames ?: return emptyList()
-        if (classNames.any { it.startsWith("java.") }) {
-            return sources
-        }
-        return classNames.map {
+        return (jcLocation?.classNames ?: return emptyList()).map {
             LazyClassSourceImpl(this, it)
         }
     }

jacodb-core/src/main/kotlin/org/jacodb/impl/storage/AbstractJcDbPersistence.kt

@@ -133,7 +133,7 @@ abstract class AbstractJcDbPersistence(
 
     override fun close() {
         try {
-            symbolInterner.setup(this)
+            symbolInterner.close()
         } catch (e: Exception) {
             // ignore
         }

jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/ErsPersistenceImpl.kt

@@ -37,6 +37,7 @@ import org.jacodb.impl.fs.PersistenceClassSource
 import org.jacodb.impl.fs.info
 import org.jacodb.impl.storage.AbstractJcDbPersistence
 import org.jacodb.impl.storage.AnnotationValueKind
+import org.jacodb.impl.storage.ers.ram.RAMEntityRelationshipStorage
 import org.jacodb.impl.storage.toJCDBContext
 import org.jacodb.impl.storage.txn
 import org.jacodb.impl.types.AnnotationInfo
@@ -75,8 +76,14 @@ class ErsPersistenceImpl(
     }
 
     override fun <T> read(action: (JCDBContext) -> T): T {
-        return ers.transactionalOptimistic(attempts = 10) { txn ->
-            action(toJCDBContext(txn))
+        return if (ers is RAMEntityRelationshipStorage) { // RAMEntityRelationshipStorage doesn't support readonly transactions
+            ers.transactionalOptimistic(attempts = 10) { txn ->
+                action(toJCDBContext(txn))
+            }
+        } else {
+            ers.transactional(readonly = true) { txn ->
+                action(toJCDBContext(txn))
+            }
         }
     }
 
@@ -178,12 +185,8 @@ class ErsPersistenceImpl(
             allClasses.forEach { classInfo ->
                 if (classInfo.superClass != null) {
                     classEntities[classInfo.name]?.let { clazz ->
-                        val inherits = links(clazz, "inherits")
                         classInfo.superClass.takeIf { JAVA_OBJECT != it }?.let { superClassName ->
-                            txn.findOrNew("SuperClass", "nameId", superClassName.asSymbolId(symbolInterner).compressed)
-                                .also { superClass ->
-                                    inherits += superClass
-                                }
+                            clazz["inherits"] = superClassName.asSymbolId(symbolInterner).compressed
                         }
                     }
                 }
@@ -196,6 +199,7 @@ class ErsPersistenceImpl(
                             txn.findOrNew("Interface", "nameId", interfaceName.asSymbolId(symbolInterner).compressed)
                                 .also { interfaceClass ->
                                     implements += interfaceClass
+                                    links(interfaceClass, "implementedBy") += clazz
                                 }
                         }
                     }

jacodb-core/src/test/kotlin/org/jacodb/testing/features/InMemoryHierarchyTest.kt

@@ -25,10 +25,13 @@ import org.jacodb.impl.features.findSubclassesInMemory
 import org.jacodb.impl.features.hierarchyExt
 import org.jacodb.impl.storage.dslContext
 import org.jacodb.impl.storage.jooq.tables.references.CLASSES
+import org.jacodb.impl.storage.txn
 import org.jacodb.testing.BaseTest
 import org.jacodb.testing.LifecycleTest
 import org.jacodb.testing.WithDB
 import org.jacodb.testing.WithGlobalDB
+import org.jacodb.testing.WithGlobalRAMDB
+import org.jacodb.testing.WithRAMDB
 import org.jacodb.testing.WithRestoredDB
 import org.junit.jupiter.api.Assertions.assertEquals
 import org.junit.jupiter.api.Assertions.assertNotNull
@@ -92,10 +95,12 @@ abstract class BaseInMemoryHierarchyTest : BaseTest() {
 
     @Test
     fun `find subclasses of Any`() {
-        val numberOfClasses = cp.db.persistence.read { it.dslContext.fetchCount(CLASSES) }
+        val numberOfClasses = getNumberOfClasses()
         assertEquals(numberOfClasses - 1, findSubClasses<Any>(allHierarchy = true).count())
     }
 
+    protected open fun getNumberOfClasses() = cp.db.persistence.read { it.dslContext.fetchCount(CLASSES) }
+
     @Test
     fun `find subclasses of Comparable`() {
         val count = findSubClasses<Comparable<*>>(allHierarchy = true).count()
@@ -133,11 +138,24 @@ class InMemoryHierarchyTest : BaseInMemoryHierarchyTest() {
     companion object : WithGlobalDB()
 }
 
+class InMemoryHierarchyRAMTest : BaseInMemoryHierarchyTest() {
+    companion object : WithGlobalRAMDB()
+
+    override fun getNumberOfClasses(): Int = cp.db.persistence.read { it.txn.all("Class").size.toInt() }
+}
+
 class RegularHierarchyTest : BaseInMemoryHierarchyTest() {
     companion object : WithDB()
 
-    override val isInMemory: Boolean
-        get() = false
+    override val isInMemory = false
+}
+
+class RegularHierarchyRAMTest : BaseInMemoryHierarchyTest() {
+    companion object : WithRAMDB()
+
+    override val isInMemory = false
+
+    override fun getNumberOfClasses(): Int = cp.db.persistence.read { it.txn.all("Class").size.toInt() }
 }
 
 @LifecycleTest