Rework Persistence API

.github/workflows/build-and-test.yml

@@ -2,6 +2,9 @@ name: Build and run tests
 
 on:
   push:
+    branches:
+      - develop
+      - neo
   pull_request:
     branches:
       - develop

buildSrc/src/main/kotlin/Dependencies.kt

@@ -8,7 +8,10 @@ object Versions {
     const val dokka = "1.7.20"
     const val gradle_download = "5.3.0"
     const val gradle_versions = "0.47.0"
-    const val hikaricp = "5.0.1"
+
+    // hikaricp version compatible with Java 8
+    const val hikaricp = "4.0.3"
+
     const val guava = "31.1-jre"
     const val javax_activation = "1.1"
     const val javax_mail = "1.4.7"
@@ -36,6 +39,9 @@ object Versions {
     const val soot_utbot_fork = "4.4.0-FORK-2"
     const val sootup = "1.0.0"
     const val sqlite = "3.41.2.2"
+    const val xodus = "2.0.1"
+    const val rocks_db = "9.1.1"
+    const val lmdb_java = "0.9.0"
 }
 
 fun dep(group: String, name: String, version: String): String = "$group:$name:$version"
@@ -275,6 +281,30 @@ object Libs {
         version = Versions.juliet
     )
 
+    val xodusUtils = dep(
+        group = "org.jetbrains.xodus",
+        name = "xodus-utils",
+        version = Versions.xodus
+    )
+
+    val xodusApi = dep(
+        group = "org.jetbrains.xodus",
+        name = "xodus-openAPI",
+        version = Versions.xodus
+    )
+
+    val xodusEnvironment = dep(
+        group = "org.jetbrains.xodus",
+        name = "xodus-environment",
+        version = Versions.xodus
+    )
+
+    val lmdb_java = dep(
+        group = "org.lmdbjava",
+        name = "lmdbjava",
+        version = Versions.lmdb_java
+    )
+
     @Suppress("FunctionName")
     fun juliet_cwe(cweNum: Int) = dep(
         group = "com.github.UnitTestBot.juliet-java-test-suite",
@@ -295,6 +325,12 @@ object Libs {
         name = "antlr4",
         version = Versions.antlr
     )
+
+    val rocks_db = dep(
+        group = "org.rocksdb",
+        name = "rocksdbjni",
+        version = Versions.rocks_db
+    )
 }
 
 object Plugins {

buildSrc/src/main/kotlin/Tests.kt

@@ -1,5 +1,6 @@
 import org.gradle.api.tasks.TaskProvider
 import org.gradle.api.tasks.testing.Test
+import java.util.*
 
 object Tests {
     val lifecycleTag = "lifecycle"
@@ -11,5 +12,15 @@ fun Test.setup(jacocoTestReport: TaskProvider<*>) {
         events("passed", "skipped", "failed")
     }
     finalizedBy(jacocoTestReport) // report is always generated after tests run
-    jvmArgs = listOf("-Xmx2g", "-XX:+HeapDumpOnOutOfMemoryError", "-XX:HeapDumpPath=heapdump.hprof")
+    val majorJavaVersion =
+        Integer.parseInt(StringTokenizer(System.getProperty("java.specification.version"), ".").nextToken())
+
+    maxHeapSize = "8G"
+
+    if (majorJavaVersion >= 16) {
+        jvmArgs = listOf(
+            "--add-opens", "java.base/java.nio=ALL-UNNAMED", // this is necessary for LMDB
+            "--add-opens", "java.base/sun.nio.ch=ALL-UNNAMED" // this is necessary for LMDB
+        )
+    }
 }

jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsNpeTest.kt

@@ -29,6 +29,7 @@ import org.jacodb.impl.features.InMemoryHierarchy
 import org.jacodb.impl.features.Usages
 import org.jacodb.impl.features.usagesExt
 import org.jacodb.testing.WithDB
+import org.jacodb.testing.WithRAMDB
 import org.jacodb.testing.analysis.NpeExamples
 import org.junit.jupiter.api.Assertions
 import org.junit.jupiter.api.Disabled
@@ -42,9 +43,9 @@ import kotlin.time.Duration.Companion.seconds
 
 private val logger = mu.KotlinLogging.logger {}
 
-class IfdsNpeTest : BaseAnalysisTest() {
+abstract class IfdsNpeTest : BaseAnalysisTest() {
 
-    companion object : WithDB(Usages, InMemoryHierarchy) {
+    companion object {
         @JvmStatic
         fun provideClassesForJuliet476(): Stream<Arguments> =
             provideClassesForJuliet(476, listOf("null_check_after_deref"))
@@ -238,3 +239,11 @@ class IfdsNpeTest : BaseAnalysisTest() {
         }
     }
 }
+
+class IfdsNpeSqlTest : IfdsNpeTest() {
+    companion object : WithDB(Usages, InMemoryHierarchy)
+}
+
+class IfdsNpeRAMTest : IfdsNpeTest() {
+    companion object : WithRAMDB(Usages, InMemoryHierarchy)
+}

jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsSqlTest.kt

@@ -28,6 +28,7 @@ import org.jacodb.api.jvm.ext.methods
 import org.jacodb.impl.features.InMemoryHierarchy
 import org.jacodb.impl.features.Usages
 import org.jacodb.testing.WithDB
+import org.jacodb.testing.WithRAMDB
 import org.jacodb.testing.analysis.SqlInjectionExamples
 import org.junit.jupiter.api.Assertions.assertTrue
 import org.junit.jupiter.api.Test
@@ -39,9 +40,9 @@ import kotlin.time.Duration.Companion.seconds
 
 private val logger = mu.KotlinLogging.logger {}
 
-class IfdsSqlTest : BaseAnalysisTest() {
+abstract class IfdsSqlTest : BaseAnalysisTest() {
 
-    companion object : WithDB(Usages, InMemoryHierarchy) {
+    companion object {
         @JvmStatic
         fun provideClassesForJuliet89(): Stream<Arguments> = provideClassesForJuliet(89, specificBansCwe89)
 
@@ -108,3 +109,11 @@ class IfdsSqlTest : BaseAnalysisTest() {
         logger.info { "SARIF:\n$sarifJson" }
     }
 }
+
+class IfdsSqlSqlTest : IfdsSqlTest() {
+    companion object : WithDB(Usages, InMemoryHierarchy)
+}
+
+class IfdsSqlRAMTest : IfdsSqlTest() {
+    companion object : WithRAMDB(Usages, InMemoryHierarchy)
+}

jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/IfdsUnusedTest.kt

@@ -23,6 +23,7 @@ import org.jacodb.api.jvm.ext.methods
 import org.jacodb.impl.features.InMemoryHierarchy
 import org.jacodb.impl.features.Usages
 import org.jacodb.testing.WithDB
+import org.jacodb.testing.WithRAMDB
 import org.junit.jupiter.api.Assertions
 import org.junit.jupiter.api.Test
 import org.junit.jupiter.params.ParameterizedTest
@@ -31,9 +32,9 @@ import org.junit.jupiter.params.provider.MethodSource
 import java.util.stream.Stream
 import kotlin.time.Duration.Companion.seconds
 
-class IfdsUnusedTest : BaseAnalysisTest() {
+abstract class IfdsUnusedTest : BaseAnalysisTest() {
 
-    companion object : WithDB(Usages, InMemoryHierarchy) {
+    companion object {
         @JvmStatic
         fun provideClassesForJuliet563(): Stream<Arguments> = provideClassesForJuliet(
             563, listOf(
@@ -77,3 +78,14 @@ class IfdsUnusedTest : BaseAnalysisTest() {
         Assertions.assertTrue(sinks.isNotEmpty())
     }
 }
+
+
+class IfdsUnusedSqlTest : IfdsUnusedTest() {
+
+    companion object : WithDB(Usages, InMemoryHierarchy)
+}
+
+class IfdsUnusedRAMTest : IfdsUnusedTest() {
+
+    companion object : WithRAMDB(Usages, InMemoryHierarchy)
+}

jacodb-analysis/src/test/kotlin/org/jacodb/analysis/impl/TaintFlowFunctionsTest.kt

@@ -43,12 +43,13 @@ import org.jacodb.taint.configuration.TaintConfigurationFeature
 import org.jacodb.taint.configuration.TaintMark
 import org.jacodb.testing.BaseTest
 import org.jacodb.testing.WithDB
+import org.jacodb.testing.WithRAMDB
 import org.jacodb.testing.allClasspath
 import org.junit.jupiter.api.Assertions
 import org.junit.jupiter.api.Test
 import org.jacodb.analysis.util.getArgument as _getArgument
 
-class TaintFlowFunctionsTest : BaseTest() {
+open class TaintFlowFunctionsTest : BaseTest() {
 
     companion object : WithDB(Usages, InMemoryHierarchy), JcTraits
 
@@ -214,3 +215,8 @@ class TaintFlowFunctionsTest : BaseTest() {
         Assertions.assertEquals(listOf(xTaint), facts)
     }
 }
+
+class TaintFlowFunctionsRAMTest : TaintFlowFunctionsTest() {
+
+    companion object : WithRAMDB(Usages, InMemoryHierarchy)
+}

jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/Api.kt

@@ -18,10 +18,13 @@ package org.jacodb.api.jvm
 
 import kotlinx.coroutines.GlobalScope
 import kotlinx.coroutines.future.future
+import org.jacodb.api.jvm.storage.ers.EntityRelationshipStorage
+import org.jacodb.api.jvm.storage.ers.Transaction
 import org.jooq.DSLContext
 import java.io.Closeable
 import java.io.File
 import java.sql.Connection
+import java.util.concurrent.ConcurrentHashMap
 
 enum class LocationType {
     RUNTIME,
@@ -142,13 +145,15 @@ interface JcDatabasePersistence : Closeable {
 
     val locations: List<JcByteCodeLocation>
 
+    val ers: EntityRelationshipStorage
+
     fun setup()
 
-    fun <T> write(action: (DSLContext) -> T): T
-    fun <T> read(action: (DSLContext) -> T): T
+    fun <T> write(action: (JCDBContext) -> T): T
+    fun <T> read(action: (JCDBContext) -> T): T
 
     fun persist(location: RegisteredLocation, classes: List<ClassSource>)
-    fun findSymbolId(symbol: String): Long?
+    fun findSymbolId(symbol: String): Long
     fun findSymbolName(symbolId: Long): String
     fun findLocation(locationId: Long): RegisteredLocation
 
@@ -162,6 +167,44 @@ interface JcDatabasePersistence : Closeable {
     fun createIndexes() {}
 }
 
+/**
+ * Abstract database access context contains several named context objects.
+ * Normally, there should be implemented specific context classes for each persistence
+ * implementation with its specific context objects.
+ *
+ * For SQLite persistence, JCDBContext should contain [DSLContext] object and may contain [Connection] object.
+ *
+ * For [EntityRelationshipStorage] persistence, JCDBContext should contain [Transaction] object.
+ */
+@Suppress("UNCHECKED_CAST")
+class JCDBContext private constructor() {
+
+    private val contextObjects = ConcurrentHashMap<ContextProperty<*>, Any>()
+
+    fun <T : Any> setContextObject(contextKey: ContextProperty<T>, contextObject: T) = apply {
+        contextObjects[contextKey] = contextObject
+    }
+
+    fun <T : Any> getContextObject(property: ContextProperty<T>): T =
+        contextObjects[property] as? T?
+            ?: throw NullPointerException("JCDBContext doesn't contain context object $property")
+
+    fun <T : Any> hasContextObject(property: ContextProperty<T>): Boolean = contextObjects.containsKey(property)
+
+    companion object {
+        fun <T : Any> of(contextKey: ContextProperty<T>, contextObject: T): JCDBContext {
+            return JCDBContext().apply { this(contextKey, contextObject) }
+        }
+
+        fun empty() = JCDBContext()
+    }
+}
+
+interface ContextProperty<T : Any>
+
+operator fun <T : Any> JCDBContext.invoke(contextKey: ContextProperty<T>, contextObject: T) =
+    setContextObject(contextKey, contextObject)
+
 interface RegisteredLocation {
     val jcLocation: JcByteCodeLocation?
     val id: Long
@@ -170,7 +213,7 @@ interface RegisteredLocation {
 }
 
 interface JCDBSymbolsInterner {
-    val jooq: DSLContext
     fun findOrNew(symbol: String): Long
-    fun flush(conn: Connection)
+    fun findSymbolName(symbolId: Long): String?
+    fun flush(context: JCDBContext)
 }

jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/Index.kt

@@ -16,15 +16,14 @@
 
 package org.jacodb.api.jvm
 
-import org.jooq.DSLContext
 import org.objectweb.asm.tree.ClassNode
 
 /** index builder */
 interface ByteCodeIndexer {
 
     fun index(classNode: ClassNode)
 
-    fun flush(jooq: DSLContext)
+    fun flush(context: JCDBContext)
 }
 
 interface JcFeature<REQ, RES> {
@@ -34,7 +33,6 @@ interface JcFeature<REQ, RES> {
     fun newIndexer(jcdb: JcDatabase, location: RegisteredLocation): ByteCodeIndexer
 
     fun onSignal(signal: JcSignal)
-
 }
 
 

jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/JcClasspath.kt

@@ -64,16 +64,16 @@ interface JcClasspath : Closeable, CommonProject {
      */
     fun findTypeOrNull(name: String): JcType?
 
-    fun classTypeOf(
+    fun typeOf(
         jcClass: JcClassOrInterface,
         nullability: Boolean? = null,
-        annotations: List<JcAnnotation> = listOf(),
-    ): JcClassType
+        annotations: List<JcAnnotation> = listOf()
+    ): JcRefType
 
     fun arrayTypeOf(
         elementType: JcType,
         nullability: Boolean? = null,
-        annotations: List<JcAnnotation> = listOf(),
+        annotations: List<JcAnnotation> = listOf()
     ): JcArrayType
 
     fun toJcClass(source: ClassSource): JcClassOrInterface
@@ -161,7 +161,7 @@ interface JcClasspathExtFeature : JcClasspathFeature {
     /**
      * semantic is the same as for `tryFindClass` method
      */
-    fun tryFindType(classpath: JcClasspath, name: String): JcResolvedTypeResult? = null
+    fun tryFindType(classpath: JcClasspath, name: String, nullable: Boolean? = null): JcResolvedTypeResult? = null
 
     fun findClasses(classpath: JcClasspath, name: String): List<JcClassOrInterface>? = null
 

jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/JcPersistenceSettings.kt

@@ -0,0 +1,28 @@
+/*
+ *  Copyright 2022 UnitTestBot contributors (utbot.org)
+ * <p>
+ *  Licensed under the Apache License, Version 2.0 (the "License");
+ *  you may not use this file except in compliance with the License.
+ *  You may obtain a copy of the License at
+ * <p>
+ *  http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ *  Unless required by applicable law or agreed to in writing, software
+ *  distributed under the License is distributed on an "AS IS" BASIS,
+ *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ *  See the License for the specific language governing permissions and
+ *  limitations under the License.
+ */
+
+package org.jacodb.api.jvm
+
+class JcPersistenceSettings {
+    val persistenceId: String? get() = implSettings?.persistenceId
+    var persistenceLocation: String? = null
+    var persistenceClearOnStart: Boolean? = null
+    var implSettings: JcPersistenceImplSettings? = null
+}
+
+interface JcPersistenceImplSettings {
+    val persistenceId: String
+}