From b945a2e040a1d4ce1f538441183c05ff2e4ee070 Mon Sep 17 00:00:00 2001 From: Valentyn Sobol Date: Wed, 14 Aug 2024 12:05:46 +0300 Subject: [PATCH] [jacodb-core] Force use of UnknownClassMethodsAndFields conditionally If the UnknownClasses classpath feature is configured to be used, then force use of the UnknownClassMethodsAndFields feature as well. [jacodb-core] Run IRTest with the UnknownClasses feature Added dependency on the commons-compress 1.21 which requires the UnknownClasses feature. [jacodb-core] Avoid using `try {} catch (e: Exception) {}` [jacodb-core] Add and use JcClasspath.registeredLocationIds [jacodb-core] Run ClassesTest with in-RAM backend --- buildSrc/src/main/kotlin/Dependencies.kt | 7 ++++ .../kotlin/org/jacodb/api/jvm/JcClasspath.kt | 1 + jacodb-core/build.gradle.kts | 1 + .../kotlin/org/jacodb/impl/JcClasspathImpl.kt | 2 +- .../kotlin/org/jacodb/impl/JcDatabaseImpl.kt | 41 +++++++++++++++---- .../org/jacodb/impl/LocationsRegistry.kt | 2 +- .../org/jacodb/impl/bytecode/KMetadata.kt | 19 ++++----- .../org/jacodb/impl/features/Builders.kt | 2 +- .../org/jacodb/impl/features/Diagnostics.kt | 2 +- .../impl/features/HierarchyExtension.kt | 6 +-- .../jacodb/impl/features/InMemoryHierarchy.kt | 2 +- .../kotlin/org/jacodb/impl/features/Usages.kt | 2 +- .../impl/storage/SQLitePersistenceImpl.kt | 3 +- .../impl/storage/ers/ErsPersistenceImpl.kt | 2 +- .../kotlin/org/jacodb/testing/ClassesTest.kt | 2 +- .../kotlin/org/jacodb/testing/cfg/IRTest.kt | 23 +++++------ .../org/jacodb/testing/LibrariesMixin.kt | 8 ++++ 17 files changed, 79 insertions(+), 46 deletions(-) diff --git a/buildSrc/src/main/kotlin/Dependencies.kt b/buildSrc/src/main/kotlin/Dependencies.kt index f1c27a427..82136e319 100644 --- a/buildSrc/src/main/kotlin/Dependencies.kt +++ b/buildSrc/src/main/kotlin/Dependencies.kt @@ -44,6 +44,7 @@ object Versions { // libs for tests only const val jgit_test_only_version = "5.9.0.202009080501-r" + const val commons_compress_test_only_version = "1.21" } fun dep(group: String, name: String, version: String): String = "$group:$name:$version" @@ -332,6 +333,12 @@ object Libs { name = "org.eclipse.jgit", version = Versions.jgit_test_only_version ) + + val commons_compress_test_only_lib = dep( + group = "org.apache.commons", + name = "commons-compress", + version = Versions.commons_compress_test_only_version + ) } object Plugins { diff --git a/jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/JcClasspath.kt b/jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/JcClasspath.kt index 680975932..a5b9f98af 100644 --- a/jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/JcClasspath.kt +++ b/jacodb-api-jvm/src/main/kotlin/org/jacodb/api/jvm/JcClasspath.kt @@ -40,6 +40,7 @@ interface JcClasspath : Closeable, CommonProject { /** locations of this classpath */ val locations: List val registeredLocations: List + val registeredLocationIds: Set val features: List? /** diff --git a/jacodb-core/build.gradle.kts b/jacodb-core/build.gradle.kts index 424ced84f..487a9592e 100644 --- a/jacodb-core/build.gradle.kts +++ b/jacodb-core/build.gradle.kts @@ -76,6 +76,7 @@ dependencies { testFixturesImplementation(Libs.jetbrains_annotations) testFixturesImplementation(Libs.kotlinx_coroutines_core) testFixturesImplementation(Libs.jgit_test_only_lib) + testFixturesImplementation(Libs.commons_compress_test_only_lib) } tasks { diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/JcClasspathImpl.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/JcClasspathImpl.kt index d10edbab9..7fe291f5b 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/JcClasspathImpl.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/JcClasspathImpl.kt @@ -48,7 +48,7 @@ class JcClasspathImpl( override val locations: List = locationsRegistrySnapshot.locations.mapNotNull { it.jcLocation } override val registeredLocations: List = locationsRegistrySnapshot.locations - + override val registeredLocationIds: Set = locationsRegistrySnapshot.ids private val classpathVfs = ClasspathVfs(globalClassVFS, locationsRegistrySnapshot) private val featuresChain = run { val strictFeatures = features.filter { it !is UnknownClasses } diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/JcDatabaseImpl.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/JcDatabaseImpl.kt index 1c14be17b..ae1b512cb 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/JcDatabaseImpl.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/JcDatabaseImpl.kt @@ -16,11 +16,31 @@ package org.jacodb.impl -import kotlinx.coroutines.* -import org.jacodb.api.jvm.* +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.async +import kotlinx.coroutines.awaitAll +import kotlinx.coroutines.cancel +import kotlinx.coroutines.delay +import kotlinx.coroutines.isActive +import kotlinx.coroutines.joinAll +import kotlinx.coroutines.launch +import kotlinx.coroutines.runBlocking +import kotlinx.coroutines.withContext +import org.jacodb.api.jvm.JavaVersion +import org.jacodb.api.jvm.JcByteCodeLocation +import org.jacodb.api.jvm.JcClasspath +import org.jacodb.api.jvm.JcClasspathFeature +import org.jacodb.api.jvm.JcDatabase +import org.jacodb.api.jvm.JcDatabasePersistence +import org.jacodb.api.jvm.JcFeature +import org.jacodb.api.jvm.RegisteredLocation import org.jacodb.impl.features.classpaths.ClasspathCache import org.jacodb.impl.features.classpaths.KotlinMetadata import org.jacodb.impl.features.classpaths.MethodInstructionsFeature +import org.jacodb.impl.features.classpaths.UnknownClassMethodsAndFields +import org.jacodb.impl.features.classpaths.UnknownClasses import org.jacodb.impl.fs.JavaRuntime import org.jacodb.impl.fs.asByteCodeLocation import org.jacodb.impl.fs.filterExisting @@ -76,14 +96,17 @@ class JcDatabaseImpl( } private fun List?.appendBuiltInFeatures(): List { - if (this != null && any { it is ClasspathCache }) { - return this + listOf(KotlinMetadata, MethodInstructionsFeature(settings.keepLocalVariableNames)) + return mutableListOf().also { result -> + result += orEmpty() + if (!result.any { it is ClasspathCache }) { + result += ClasspathCache(settings.cacheSettings) + } + result += KotlinMetadata + result += MethodInstructionsFeature(settings.keepLocalVariableNames) + if (result.any { it is UnknownClasses } && !result.any { it is UnknownClassMethodsAndFields }) { + result += UnknownClassMethodsAndFields + } } - return listOf( - ClasspathCache(settings.cacheSettings), - KotlinMetadata, - MethodInstructionsFeature(settings.keepLocalVariableNames) - ) + orEmpty() } override suspend fun classpath(dirOrJars: List, features: List?): JcClasspath { diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/LocationsRegistry.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/LocationsRegistry.kt index 6854126b4..f38b37171 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/LocationsRegistry.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/LocationsRegistry.kt @@ -55,7 +55,7 @@ open class LocationsRegistrySnapshot( val locations: List ) : Closeable { - val ids = locations.map { it.id }.toHashSet() + val ids = locations.mapTo(mutableSetOf()) { it.id } override fun close() = registry.close(this) diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/bytecode/KMetadata.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/bytecode/KMetadata.kt index fbf1cca4f..050e4ec23 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/bytecode/KMetadata.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/bytecode/KMetadata.kt @@ -47,20 +47,15 @@ val JcMethod.kmConstructor: KmConstructor? val JcParameter.kmParameter: KmValueParameter? get() { - try { - method.kmFunction?.let { - // Shift needed to properly handle extension functions - val shift = if (it.receiverParameterType != null) 1 else 0 + method.kmFunction?.let { + // Shift needed to properly handle extension functions + val shift = if (it.receiverParameterType != null) 1 else 0 - // index - shift could be out of bounds if generated JVM parameter is fictive - // E.g., see how extension functions and coroutines are compiled - return it.valueParameters.getOrNull(index - shift) - } - - return method.kmConstructor?.valueParameters?.get(index) - } catch (e: Exception) { - return null + // index - shift could be out of bounds if generated JVM parameter is fictive + // E.g., see how extension functions and coroutines are compiled + return it.valueParameters.getOrNull(index - shift) } + return method.kmConstructor?.valueParameters?.getOrNull(index) } // If parameter is a receiver parameter, it doesn't have KmValueParameter instance, but we still can get KmType for it diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/Builders.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/Builders.kt index 773a8b077..e58672c75 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/Builders.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/Builders.kt @@ -237,7 +237,7 @@ object Builders : JcFeature, BuildersResponse> { } fun syncQuery(classpath: JcClasspath, req: Set): Sequence { - val locationIds = classpath.registeredLocations.map { it.id } + val locationIds = classpath.registeredLocationIds val persistence = classpath.db.persistence return sequence { val result = persistence.read { context -> diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/Diagnostics.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/Diagnostics.kt index 48339abca..95215cf63 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/Diagnostics.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/Diagnostics.kt @@ -39,7 +39,7 @@ suspend fun JcClasspath.duplicatedClasses(): Map { sqlAction = { jooq -> jooq.select(SYMBOLS.NAME, DSL.count(SYMBOLS.NAME)).from(CLASSES) .join(SYMBOLS).on(SYMBOLS.ID.eq(CLASSES.NAME)) - .where(CLASSES.LOCATION_ID.`in`(registeredLocations.map { it.id })) + .where(CLASSES.LOCATION_ID.`in`(registeredLocationIds)) .groupBy(SYMBOLS.NAME) .having(DSL.count(SYMBOLS.NAME).greaterThan(1)) .fetch() diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/HierarchyExtension.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/HierarchyExtension.kt index d0f94c5c3..faac1c373 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/HierarchyExtension.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/HierarchyExtension.kt @@ -63,7 +63,7 @@ suspend fun JcClasspath.hierarchyExt(): HierarchyExtension { fun JcClasspath.asyncHierarchyExt(): Future = GlobalScope.future { hierarchyExt() } internal fun JcClasspath.allClassesExceptObject(context: JCDBContext, direct: Boolean): Sequence { - val locationIds = registeredLocations.mapTo(mutableSetOf()) { it.id } + val locationIds = registeredLocationIds return context.execute( sqlAction = { jooq -> if (direct) { @@ -162,7 +162,7 @@ private class HierarchyExtensionERS(cp: JcClasspath) : HierarchyExtensionBase(cp if (name == JAVA_OBJECT) { cp.allClassesExceptObject(context, !entireHierarchy) } else { - val locationIds = cp.registeredLocations.mapTo(mutableSetOf()) { it.id } + val locationIds = cp.registeredLocationIds val nameId = name.asSymbolId(persistence.symbolInterner) if (entireHierarchy) { entireHierarchy(txn, nameId, mutableSetOf()) @@ -251,7 +251,7 @@ private fun JcClasspath.subClasses(name: String, entireHierarchy: Boolean): Sequ if (name == JAVA_OBJECT) { allClassesExceptObject(context, !entireHierarchy) } else { - val locationIds = registeredLocations.joinToString(", ") { it.id.toString() } + val locationIds = registeredLocationIds.joinToString(", ") { it.toString() } val dslContext = context.dslContext BatchedSequence(defaultBatchSize) { offset, batchSize -> val query = when { diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/InMemoryHierarchy.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/InMemoryHierarchy.kt index bcd16915c..aad98739a 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/InMemoryHierarchy.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/InMemoryHierarchy.kt @@ -188,7 +188,7 @@ object InMemoryHierarchy : JcFeature { } } - val locationIds = classpath.registeredLocations.mapTo(mutableSetOf()) { it.id } + val locationIds = classpath.registeredLocationIds val classSymbolId = persistence.findSymbolId(req.name) val allSubclasses = hashSetOf() diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/Usages.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/Usages.kt index 5639557d9..0a3e993f6 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/features/Usages.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/features/Usages.kt @@ -200,7 +200,7 @@ object Usages : JcFeature { } fun syncQuery(classpath: JcClasspath, req: UsageFeatureRequest): Sequence { - val locationIds = classpath.registeredLocations.mapTo(mutableSetOf()) { it.id } + val locationIds = classpath.registeredLocationIds val persistence = classpath.db.persistence val symbolInterner = persistence.symbolInterner val name = req.methodName ?: req.field ?: throw IllegalArgumentException("Callee name should be specified") diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/SQLitePersistenceImpl.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/SQLitePersistenceImpl.kt index 1381d5781..ae52fa284 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/SQLitePersistenceImpl.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/SQLitePersistenceImpl.kt @@ -113,8 +113,7 @@ class SQLitePersistenceImpl( private val JcClasspath.clause: Condition get() { - val ids = registeredLocations.map { it.id } - return CLASSES.LOCATION_ID.`in`(ids) + return CLASSES.LOCATION_ID.`in`(registeredLocationIds) } private fun JcDatabase.classSources(clause: Condition, single: Boolean = false): List = read { context -> diff --git a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/ErsPersistenceImpl.kt b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/ErsPersistenceImpl.kt index 669540877..fce6cec14 100644 --- a/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/ErsPersistenceImpl.kt +++ b/jacodb-core/src/main/kotlin/org/jacodb/impl/storage/ers/ErsPersistenceImpl.kt @@ -243,7 +243,7 @@ class ErsPersistenceImpl( } private fun findClassSourcesImpl(context: JCDBContext, cp: JcClasspath, fullName: String): Sequence { - val ids = cp.registeredLocations.mapTo(hashSetOf()) { it.id } + val ids = cp.registeredLocationIds return context.txn.find("Class", "nameId", findSymbolId(fullName).compressed) .asSequence().filter { it.getCompressed("locationId") in ids } .map { it.toClassSource(cp.db, fullName) } diff --git a/jacodb-core/src/test/kotlin/org/jacodb/testing/ClassesTest.kt b/jacodb-core/src/test/kotlin/org/jacodb/testing/ClassesTest.kt index 2f74ccbbf..b353acda0 100644 --- a/jacodb-core/src/test/kotlin/org/jacodb/testing/ClassesTest.kt +++ b/jacodb-core/src/test/kotlin/org/jacodb/testing/ClassesTest.kt @@ -33,7 +33,7 @@ import org.junit.jupiter.api.Test class ClassesTest : DatabaseEnvTest() { - companion object : WithGlobalDB() + companion object : WithGlobalRAMDB() override val cp: JcClasspath = runBlocking { db.classpath(allClasspath) } diff --git a/jacodb-core/src/test/kotlin/org/jacodb/testing/cfg/IRTest.kt b/jacodb-core/src/test/kotlin/org/jacodb/testing/cfg/IRTest.kt index 2ca6df61c..c7de7cdec 100644 --- a/jacodb-core/src/test/kotlin/org/jacodb/testing/cfg/IRTest.kt +++ b/jacodb-core/src/test/kotlin/org/jacodb/testing/cfg/IRTest.kt @@ -16,13 +16,6 @@ package org.jacodb.testing.cfg -import org.jacodb.api.common.cfg.CommonAssignInst -import org.jacodb.api.common.cfg.CommonCallInst -import org.jacodb.api.common.cfg.CommonExpr -import org.jacodb.api.common.cfg.CommonGotoInst -import org.jacodb.api.common.cfg.CommonIfInst -import org.jacodb.api.common.cfg.CommonInst -import org.jacodb.api.common.cfg.CommonReturnInst import org.jacodb.api.jvm.JavaVersion import org.jacodb.api.jvm.JcClassType import org.jacodb.api.jvm.JcMethod @@ -61,10 +54,12 @@ import org.jacodb.impl.cfg.Simplifier import org.jacodb.impl.cfg.util.ExprMapper import org.jacodb.impl.features.classpaths.ClasspathCache import org.jacodb.impl.features.classpaths.StringConcatSimplifier +import org.jacodb.impl.features.classpaths.UnknownClasses import org.jacodb.impl.fs.JarLocation import org.jacodb.testing.WithDB import org.jacodb.testing.WithRAMDB import org.jacodb.testing.asmLib +import org.jacodb.testing.commonsCompressLib import org.jacodb.testing.guavaLib import org.jacodb.testing.jgitLib import org.jacodb.testing.kotlinStdLib @@ -73,7 +68,6 @@ import org.junit.jupiter.api.AfterEach import org.junit.jupiter.api.Assertions.assertEquals import org.junit.jupiter.api.Assertions.assertFalse import org.junit.jupiter.api.Assertions.assertTrue -import org.junit.jupiter.api.Disabled import org.junit.jupiter.api.Test import org.junit.jupiter.api.assertDoesNotThrow import java.io.File @@ -95,8 +89,8 @@ class OverridesResolver( return methods.firstOrNull { typedMethod -> val jcMethod = typedMethod.method jcMethod.name == name && - jcMethod.returnType.typeName == returnType.typeName && - jcMethod.parameters.map { param -> param.type.typeName } == argTypes.map { it.typeName } + jcMethod.returnType.typeName == returnType.typeName && + jcMethod.parameters.map { param -> param.type.typeName } == argTypes.map { it.typeName } } ?: error("Could not find a method with correct signature") } @@ -350,6 +344,11 @@ abstract class IRTest : BaseInstructionsTest() { runAlongLib(jgitLib) } + @Test + fun `get ir of commons compress`() { + runAlongLib(commonsCompressLib) + } + @Test fun `get ir of asm`() { runAlongLib(asmLib, muteGraphChecker = true) @@ -392,10 +391,10 @@ abstract class IRTest : BaseInstructionsTest() { class IRSqlTest : IRTest() { - companion object : WithDB(StringConcatSimplifier) + companion object : WithDB(StringConcatSimplifier, UnknownClasses) } class IRRAMTest : IRTest() { - companion object : WithRAMDB(StringConcatSimplifier) + companion object : WithRAMDB(StringConcatSimplifier, UnknownClasses) } \ No newline at end of file diff --git a/jacodb-core/src/testFixtures/kotlin/org/jacodb/testing/LibrariesMixin.kt b/jacodb-core/src/testFixtures/kotlin/org/jacodb/testing/LibrariesMixin.kt index b78aeec66..53ebb9b1e 100644 --- a/jacodb-core/src/testFixtures/kotlin/org/jacodb/testing/LibrariesMixin.kt +++ b/jacodb-core/src/testFixtures/kotlin/org/jacodb/testing/LibrariesMixin.kt @@ -41,6 +41,14 @@ val jgitLib: File } } +val commonsCompressLib: File + get() { + val commonsCompressUrl = classpath.first { it.contains("commons-compress-") } + return File(commonsCompressUrl).also { + Assertions.assertTrue(it.isFile && it.exists()) + } + } + val asmLib: File get() { val asmUrl = classpath.first { it.contains("${File.separator}asm${File.separator}") }